codemie-test-harness 0.1.184__py3-none-any.whl → 0.1.197__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of codemie-test-harness might be problematic. Click here for more details.
- codemie_test_harness/cli/cli.py +21 -6
- codemie_test_harness/cli/commands/run_cmd.py +15 -1
- codemie_test_harness/cli/constants.py +1 -21
- codemie_test_harness/cli/runner.py +13 -3
- codemie_test_harness/cli/utils.py +12 -2
- codemie_test_harness/tests/assistant/test_assistants.py +17 -1
- codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +47 -6
- codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +0 -4
- codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +0 -4
- codemie_test_harness/tests/conftest.py +2 -2
- codemie_test_harness/tests/enums/environment.py +3 -3
- codemie_test_harness/tests/enums/integrations.py +1 -0
- codemie_test_harness/tests/enums/model_types.py +1 -0
- codemie_test_harness/tests/integrations/project/test_default_integrations.py +41 -15
- codemie_test_harness/tests/integrations/project/test_project_integrations.py +42 -0
- codemie_test_harness/tests/integrations/user/test_default_integrations.py +0 -7
- codemie_test_harness/tests/llm/assistants/test_llm.py +56 -2
- codemie_test_harness/tests/test_data/assistant_test_data.py +448 -448
- codemie_test_harness/tests/test_data/codebase_tools_test_data.py +2 -0
- codemie_test_harness/tests/test_data/data_management_tools_test_data.py +18 -0
- codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py +18 -1
- codemie_test_harness/tests/test_data/direct_tools/report_portal_tools_test_data.py +189 -197
- codemie_test_harness/tests/test_data/integrations_test_data.py +163 -2
- codemie_test_harness/tests/test_data/llm_test_data.py +1 -0
- codemie_test_harness/tests/test_data/open_api_tools_test_data.py +22 -1
- codemie_test_harness/tests/test_data/report_portal_tools_test_data.py +89 -112
- codemie_test_harness/tests/test_data/research_tools_test_data.py +29 -7
- codemie_test_harness/tests/utils/assistant_utils.py +22 -12
- codemie_test_harness/tests/utils/credentials_manager.py +66 -8
- codemie_test_harness/tests/utils/workflow_utils.py +91 -0
- codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +50 -1
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +39 -1
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +34 -1
- codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +11 -0
- codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +72 -1
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +31 -0
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +31 -0
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +26 -0
- codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +63 -1
- codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +82 -7
- codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +23 -4
- codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +12 -0
- codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +29 -2
- codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +14 -1
- codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/report_portal/test_workflow_with_assistant_with_report_portal_tools.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +14 -1
- codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +6 -0
- codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +12 -9
- codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +8 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +28 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +24 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +20 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +13 -4
- codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +16 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +73 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +101 -49
- codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +42 -3
- codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +27 -5
- codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +13 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +20 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +14 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +9 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +9 -1
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.197.dist-info}/METADATA +38 -3
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.197.dist-info}/RECORD +79 -79
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.197.dist-info}/WHEEL +0 -0
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.197.dist-info}/entry_points.txt +0 -0
codemie_test_harness/cli/cli.py
CHANGED
|
@@ -15,6 +15,7 @@ from .constants import (
|
|
|
15
15
|
KEY_MARKS,
|
|
16
16
|
KEY_XDIST_N,
|
|
17
17
|
KEY_RERUNS,
|
|
18
|
+
KEY_COUNT,
|
|
18
19
|
KEY_AUTH_SERVER_URL,
|
|
19
20
|
KEY_AUTH_CLIENT_ID,
|
|
20
21
|
KEY_AUTH_CLIENT_SECRET,
|
|
@@ -22,12 +23,9 @@ from .constants import (
|
|
|
22
23
|
KEY_CODEMIE_API_DOMAIN,
|
|
23
24
|
KEY_AUTH_USERNAME,
|
|
24
25
|
KEY_AUTH_PASSWORD,
|
|
25
|
-
# integrations
|
|
26
26
|
DEFAULT_MARKS,
|
|
27
27
|
DEFAULT_XDIST_N,
|
|
28
28
|
DEFAULT_RERUNS,
|
|
29
|
-
AUTH_KEYS,
|
|
30
|
-
INTEGRATION_KEYS,
|
|
31
29
|
)
|
|
32
30
|
from .utils import get_config_value, ensure_env_from_config
|
|
33
31
|
from .runner import run_pytest
|
|
@@ -50,6 +48,12 @@ from .commands.marks_cmd import marks_cmd
|
|
|
50
48
|
@click.option(
|
|
51
49
|
"--reruns", envvar=KEY_RERUNS, type=int, help="Number of reruns for flaky tests"
|
|
52
50
|
)
|
|
51
|
+
@click.option(
|
|
52
|
+
"--count",
|
|
53
|
+
envvar=KEY_COUNT,
|
|
54
|
+
type=int,
|
|
55
|
+
help="Number of times to repeat each test (requires pytest-repeat)",
|
|
56
|
+
)
|
|
53
57
|
@click.option("--auth-server-url", envvar=KEY_AUTH_SERVER_URL, help="Auth server url")
|
|
54
58
|
@click.option("--auth-client-id", envvar=KEY_AUTH_CLIENT_ID, help="Auth client id")
|
|
55
59
|
@click.option(
|
|
@@ -68,6 +72,7 @@ def cli(
|
|
|
68
72
|
marks: Optional[str],
|
|
69
73
|
workers: Optional[int],
|
|
70
74
|
reruns: Optional[int],
|
|
75
|
+
count: Optional[int],
|
|
71
76
|
auth_server_url: Optional[str],
|
|
72
77
|
auth_client_id: Optional[str],
|
|
73
78
|
auth_client_secret: Optional[str],
|
|
@@ -99,6 +104,11 @@ def cli(
|
|
|
99
104
|
if reruns is not None
|
|
100
105
|
else int(get_config_value(KEY_RERUNS, str(DEFAULT_RERUNS)))
|
|
101
106
|
)
|
|
107
|
+
resolved_count = (
|
|
108
|
+
count
|
|
109
|
+
if count is not None
|
|
110
|
+
else (int(get_config_value(KEY_COUNT)) if get_config_value(KEY_COUNT) else None)
|
|
111
|
+
)
|
|
102
112
|
|
|
103
113
|
# Ensure env vars. CLI args override env/config.
|
|
104
114
|
provided = {
|
|
@@ -115,15 +125,20 @@ def cli(
|
|
|
115
125
|
if v is not None and v != "":
|
|
116
126
|
os.environ[k] = str(v)
|
|
117
127
|
# populate any missing values from saved config
|
|
118
|
-
ensure_env_from_config(
|
|
128
|
+
ensure_env_from_config()
|
|
119
129
|
|
|
120
130
|
ctx.obj.update(
|
|
121
|
-
dict(
|
|
131
|
+
dict(
|
|
132
|
+
marks=resolved_marks,
|
|
133
|
+
workers=resolved_workers,
|
|
134
|
+
reruns=resolved_reruns,
|
|
135
|
+
count=resolved_count,
|
|
136
|
+
)
|
|
122
137
|
)
|
|
123
138
|
|
|
124
139
|
# default behavior
|
|
125
140
|
if ctx.invoked_subcommand is None and not ctx.resilient_parsing:
|
|
126
|
-
run_pytest(resolved_workers, resolved_marks, resolved_reruns)
|
|
141
|
+
run_pytest(resolved_workers, resolved_marks, resolved_reruns, resolved_count)
|
|
127
142
|
|
|
128
143
|
|
|
129
144
|
# Register subcommands
|
|
@@ -10,6 +10,11 @@ from ..runner import run_pytest
|
|
|
10
10
|
"-n", "workers", type=int, help="Override number of xdist workers for this run"
|
|
11
11
|
)
|
|
12
12
|
@click.option("--reruns", type=int, help="Override number of reruns for this run")
|
|
13
|
+
@click.option(
|
|
14
|
+
"--count",
|
|
15
|
+
type=int,
|
|
16
|
+
help="Number of times to repeat each test (requires pytest-repeat)",
|
|
17
|
+
)
|
|
13
18
|
@click.argument("extra", nargs=-1)
|
|
14
19
|
@click.pass_context
|
|
15
20
|
def run_cmd(
|
|
@@ -17,14 +22,23 @@ def run_cmd(
|
|
|
17
22
|
marks: Optional[str],
|
|
18
23
|
workers: Optional[int],
|
|
19
24
|
reruns: Optional[int],
|
|
25
|
+
count: Optional[int],
|
|
20
26
|
extra: Tuple[str, ...],
|
|
21
27
|
):
|
|
22
28
|
"""Run pytest with configured options.
|
|
23
29
|
|
|
24
30
|
Example: codemie-test-harness run --marks "smoke and not ui" -n 8 --reruns 2 -k keyword
|
|
31
|
+
Example with repeat: codemie-test-harness run --marks excel_generation --count 50 -n 10
|
|
25
32
|
"""
|
|
26
33
|
resolved_marks = marks or ctx.obj.get("marks")
|
|
27
34
|
resolved_workers = workers if workers is not None else ctx.obj.get("workers")
|
|
28
35
|
resolved_reruns = reruns if reruns is not None else ctx.obj.get("reruns")
|
|
36
|
+
resolved_count = count if count is not None else ctx.obj.get("count")
|
|
29
37
|
|
|
30
|
-
run_pytest(
|
|
38
|
+
run_pytest(
|
|
39
|
+
int(resolved_workers),
|
|
40
|
+
str(resolved_marks),
|
|
41
|
+
int(resolved_reruns),
|
|
42
|
+
resolved_count,
|
|
43
|
+
extra,
|
|
44
|
+
)
|
|
@@ -19,6 +19,7 @@ KEY_CODEMIE_API_DOMAIN = "CODEMIE_API_DOMAIN"
|
|
|
19
19
|
KEY_MARKS = "PYTEST_MARKS"
|
|
20
20
|
KEY_XDIST_N = "PYTEST_N"
|
|
21
21
|
KEY_RERUNS = "PYTEST_RERUNS"
|
|
22
|
+
KEY_COUNT = "PYTEST_COUNT"
|
|
22
23
|
|
|
23
24
|
# === COMPLETE INTEGRATION CREDENTIALS KEYS ===
|
|
24
25
|
# Version Control Systems (GitLab, GitHub)
|
|
@@ -146,16 +147,6 @@ DEFAULT_MARKS = "smoke"
|
|
|
146
147
|
DEFAULT_XDIST_N = 8
|
|
147
148
|
DEFAULT_RERUNS = 2
|
|
148
149
|
|
|
149
|
-
AUTH_KEYS = [
|
|
150
|
-
KEY_AUTH_SERVER_URL,
|
|
151
|
-
KEY_AUTH_CLIENT_ID,
|
|
152
|
-
KEY_AUTH_CLIENT_SECRET,
|
|
153
|
-
KEY_AUTH_USERNAME,
|
|
154
|
-
KEY_AUTH_PASSWORD,
|
|
155
|
-
KEY_AUTH_REALM_NAME,
|
|
156
|
-
KEY_CODEMIE_API_DOMAIN,
|
|
157
|
-
]
|
|
158
|
-
|
|
159
150
|
# === CREDENTIAL CATEGORIES ===
|
|
160
151
|
VERSION_CONTROL_KEYS = [
|
|
161
152
|
KEY_GIT_ENV,
|
|
@@ -283,17 +274,6 @@ DATA_MANAGEMENT_KEYS = [
|
|
|
283
274
|
KEY_MSSQL_PASSWORD,
|
|
284
275
|
]
|
|
285
276
|
|
|
286
|
-
# Combined lists for backwards compatibility
|
|
287
|
-
INTEGRATION_KEYS = (
|
|
288
|
-
VERSION_CONTROL_KEYS
|
|
289
|
-
+ PROJECT_MANAGEMENT_KEYS
|
|
290
|
-
+ CLOUD_PROVIDERS_KEYS
|
|
291
|
-
+ DEVELOPMENT_TOOLS_KEYS
|
|
292
|
-
+ NOTIFICATIONS_KEYS
|
|
293
|
-
+ RESEARCH_TOOLS_KEYS
|
|
294
|
-
+ DATA_MANAGEMENT_KEYS
|
|
295
|
-
)
|
|
296
|
-
|
|
297
277
|
# Category mapping for CLI
|
|
298
278
|
CREDENTIAL_CATEGORIES = {
|
|
299
279
|
"version-control": {
|
|
@@ -49,7 +49,11 @@ def resolve_tests_path_and_root() -> tuple[str, str]:
|
|
|
49
49
|
|
|
50
50
|
|
|
51
51
|
def build_pytest_cmd(
|
|
52
|
-
workers: int,
|
|
52
|
+
workers: int,
|
|
53
|
+
marks: str,
|
|
54
|
+
reruns: int,
|
|
55
|
+
count: int | None = None,
|
|
56
|
+
extra: Iterable[str] | None = None,
|
|
53
57
|
) -> tuple[List[str], str]:
|
|
54
58
|
tests_path, root_dir = resolve_tests_path_and_root()
|
|
55
59
|
cmd = [sys.executable, "-m", "pytest", tests_path]
|
|
@@ -59,6 +63,8 @@ def build_pytest_cmd(
|
|
|
59
63
|
cmd += ["-m", str(marks)]
|
|
60
64
|
if reruns and int(reruns) > 0:
|
|
61
65
|
cmd += ["--reruns", str(reruns)]
|
|
66
|
+
if count and int(count) > 0:
|
|
67
|
+
cmd += ["--count", str(count)]
|
|
62
68
|
if extra:
|
|
63
69
|
cmd += list(extra)
|
|
64
70
|
return cmd, root_dir
|
|
@@ -97,11 +103,15 @@ def validate_marks_expression(marks: str) -> None:
|
|
|
97
103
|
|
|
98
104
|
|
|
99
105
|
def run_pytest(
|
|
100
|
-
workers: int,
|
|
106
|
+
workers: int,
|
|
107
|
+
marks: str,
|
|
108
|
+
reruns: int,
|
|
109
|
+
count: int | None = None,
|
|
110
|
+
extra: Iterable[str] | None = None,
|
|
101
111
|
) -> None:
|
|
102
112
|
# Validate marks before running pytest
|
|
103
113
|
validate_marks_expression(marks)
|
|
104
114
|
|
|
105
|
-
cmd, root_dir = build_pytest_cmd(workers, marks, reruns, extra)
|
|
115
|
+
cmd, root_dir = build_pytest_cmd(workers, marks, reruns, count, extra)
|
|
106
116
|
CONSOLE.print(f"[cyan]Running:[/] {' '.join(cmd)} (cwd={root_dir})")
|
|
107
117
|
raise SystemExit(subprocess.call(cmd, cwd=root_dir))
|
|
@@ -37,9 +37,19 @@ def set_config_value(key: str, value: Any) -> None:
|
|
|
37
37
|
save_config(cfg)
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def ensure_env_from_config(keys: list[str]) -> None:
|
|
41
|
-
|
|
40
|
+
def ensure_env_from_config(keys: Optional[list[str]] = None) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Populate missing environment variables from config file.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
keys: Optional list of specific keys to load. If None, loads all keys from config.
|
|
46
|
+
"""
|
|
42
47
|
cfg = load_config()
|
|
48
|
+
|
|
49
|
+
# If no specific keys provided, load all keys from config
|
|
50
|
+
if keys is None:
|
|
51
|
+
keys = list(cfg.keys())
|
|
52
|
+
|
|
43
53
|
for k in keys:
|
|
44
54
|
if k not in os.environ and k in cfg:
|
|
45
55
|
os.environ[k] = str(cfg[k])
|
|
@@ -436,7 +436,9 @@ def test_excel_tool_extended_functionality(
|
|
|
436
436
|
- Multi-sheet comprehensive analysis
|
|
437
437
|
|
|
438
438
|
"""
|
|
439
|
-
assistant_instance = assistant(
|
|
439
|
+
assistant_instance = assistant(
|
|
440
|
+
system_prompt="You have all required information in initial prompt. Do not ask additional questions and proceed with request."
|
|
441
|
+
)
|
|
440
442
|
|
|
441
443
|
uploaded_file = assistant_utils.upload_file_to_chat(
|
|
442
444
|
FILES_PATH / "test_extended.xlsx"
|
|
@@ -490,3 +492,17 @@ def test_docx_tool_extended_functionality(
|
|
|
490
492
|
|
|
491
493
|
assert_tool_triggered(Default.DOCX_TOOL, triggered_tools)
|
|
492
494
|
similarity_check.check_similarity(response, expected_response)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
@pytest.mark.assistant
|
|
498
|
+
# @pytest.mark.api
|
|
499
|
+
@pytest.mark.smoke
|
|
500
|
+
@pytest.mark.excel_generation
|
|
501
|
+
def test_excel_file_generation(assistant_utils, assistant, similarity_check):
|
|
502
|
+
assistant_instance = assistant()
|
|
503
|
+
|
|
504
|
+
response = assistant_utils.ask_assistant(
|
|
505
|
+
assistant_instance, "Generate excel with 5 sheets with random data"
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
similarity_check.check_similarity(response, "expected_response_here")
|
|
@@ -3,6 +3,7 @@ import uuid
|
|
|
3
3
|
import pytest
|
|
4
4
|
from codemie_sdk.models.integration import CredentialTypes
|
|
5
5
|
|
|
6
|
+
from codemie_test_harness.tests.enums.integrations import DataBaseDialect
|
|
6
7
|
from codemie_test_harness.tests.enums.tools import (
|
|
7
8
|
Toolkit,
|
|
8
9
|
DataManagementTool,
|
|
@@ -16,10 +17,12 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
|
|
|
16
17
|
SQL_TOOL_INSERT_TABLE_TASK,
|
|
17
18
|
SQL_TOOL_QUERY_TABLE_TASK,
|
|
18
19
|
RESPONSE_FOR_SQL,
|
|
20
|
+
INFLUXDB_QUERY_MEASUREMENT_TASK,
|
|
21
|
+
RESPONSE_FOR_INFLUXDB,
|
|
19
22
|
)
|
|
23
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
20
24
|
from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
|
|
21
25
|
from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
|
|
22
|
-
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
23
26
|
|
|
24
27
|
pytestmark = pytest.mark.skipif(
|
|
25
28
|
EnvironmentResolver.is_localhost(),
|
|
@@ -70,12 +73,15 @@ def test_create_assistant_with_sql_tool(
|
|
|
70
73
|
)
|
|
71
74
|
|
|
72
75
|
assistant = assistant(
|
|
73
|
-
Toolkit.DATA_MANAGEMENT,
|
|
76
|
+
Toolkit.DATA_MANAGEMENT,
|
|
77
|
+
DataManagementTool.SQL,
|
|
78
|
+
settings=settings,
|
|
79
|
+
system_prompt="Always run tools for user prompt",
|
|
74
80
|
)
|
|
75
81
|
|
|
76
82
|
conversation_id = str(uuid.uuid4())
|
|
77
83
|
|
|
78
|
-
|
|
84
|
+
_, triggered_tools = assistant_utils.ask_assistant(
|
|
79
85
|
assistant,
|
|
80
86
|
SQL_TOOL_CREATE_TABLE_TASK,
|
|
81
87
|
conversation_id=conversation_id,
|
|
@@ -83,7 +89,7 @@ def test_create_assistant_with_sql_tool(
|
|
|
83
89
|
)
|
|
84
90
|
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
85
91
|
|
|
86
|
-
|
|
92
|
+
_, triggered_tools = assistant_utils.ask_assistant(
|
|
87
93
|
assistant,
|
|
88
94
|
SQL_TOOL_INSERT_TABLE_TASK,
|
|
89
95
|
conversation_id=conversation_id,
|
|
@@ -98,8 +104,9 @@ def test_create_assistant_with_sql_tool(
|
|
|
98
104
|
minimal_response=False,
|
|
99
105
|
)
|
|
100
106
|
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
107
|
+
similarity_check.check_similarity(response, RESPONSE_FOR_SQL)
|
|
101
108
|
|
|
102
|
-
|
|
109
|
+
_, triggered_tools = assistant_utils.ask_assistant(
|
|
103
110
|
assistant,
|
|
104
111
|
SQL_TOOL_DELETE_TABLE_TASK,
|
|
105
112
|
conversation_id=conversation_id,
|
|
@@ -107,4 +114,38 @@ def test_create_assistant_with_sql_tool(
|
|
|
107
114
|
)
|
|
108
115
|
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
109
116
|
|
|
110
|
-
|
|
117
|
+
|
|
118
|
+
@pytest.mark.assistant
|
|
119
|
+
@pytest.mark.sql
|
|
120
|
+
@pytest.mark.influx
|
|
121
|
+
@pytest.mark.api
|
|
122
|
+
@pytest.mark.testcase("EPMCDME-6132")
|
|
123
|
+
@pytest.mark.skipif(
|
|
124
|
+
not EnvironmentResolver.is_sandbox(),
|
|
125
|
+
reason="InfluxDB is only available in sandbox environments",
|
|
126
|
+
)
|
|
127
|
+
def test_create_assistant_with_influxdb_tool(
|
|
128
|
+
integration_utils, assistant_utils, assistant, similarity_check
|
|
129
|
+
):
|
|
130
|
+
"""Test creating assistant with InfluxDB tool and performing time-series operations."""
|
|
131
|
+
|
|
132
|
+
credential_values = CredentialsManager.sql_credentials(DataBaseDialect.INFLUX)
|
|
133
|
+
settings = integration_utils.create_integration(
|
|
134
|
+
CredentialTypes.SQL, credential_values
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
assistant = assistant(
|
|
138
|
+
Toolkit.DATA_MANAGEMENT, DataManagementTool.SQL, settings=settings
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
conversation_id = str(uuid.uuid4())
|
|
142
|
+
|
|
143
|
+
# Query the measurement
|
|
144
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
145
|
+
assistant,
|
|
146
|
+
INFLUXDB_QUERY_MEASUREMENT_TASK,
|
|
147
|
+
conversation_id=conversation_id,
|
|
148
|
+
minimal_response=False,
|
|
149
|
+
)
|
|
150
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
151
|
+
similarity_check.check_similarity(response, RESPONSE_FOR_INFLUXDB)
|
|
@@ -17,10 +17,6 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
|
|
|
17
17
|
CLI_MCP_SERVER,
|
|
18
18
|
)
|
|
19
19
|
|
|
20
|
-
# pytestmark = pytest.mark.skipif(
|
|
21
|
-
# EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
|
|
22
|
-
# )
|
|
23
|
-
|
|
24
20
|
|
|
25
21
|
@pytest.mark.assistant
|
|
26
22
|
@pytest.mark.mcp
|
|
@@ -12,10 +12,6 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
|
|
|
12
12
|
fetch_server_prompt,
|
|
13
13
|
)
|
|
14
14
|
|
|
15
|
-
# pytestmark = pytest.mark.skipif(
|
|
16
|
-
# EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
|
|
17
|
-
# )
|
|
18
|
-
|
|
19
15
|
|
|
20
16
|
@pytest.mark.assistant
|
|
21
17
|
@pytest.mark.mcp
|
|
@@ -845,8 +845,8 @@ def ado_integration(integration_utils):
|
|
|
845
845
|
|
|
846
846
|
|
|
847
847
|
@pytest.fixture(scope="function")
|
|
848
|
-
def
|
|
849
|
-
"""Create
|
|
848
|
+
def integration(integration_utils):
|
|
849
|
+
"""Create integration with custom credentials"""
|
|
850
850
|
|
|
851
851
|
created_integration: Optional[Integration] = None
|
|
852
852
|
|
|
@@ -81,7 +81,7 @@ class Environment(Enum):
|
|
|
81
81
|
Returns:
|
|
82
82
|
List of Environment enums: [PREVIEW, AZURE, LOCALHOST]
|
|
83
83
|
"""
|
|
84
|
-
return [cls.PREVIEW, cls.AZURE, cls.LOCALHOST]
|
|
84
|
+
return [cls.PREVIEW, cls.AZURE, cls.LOCALHOST, cls.PRODUCTION]
|
|
85
85
|
|
|
86
86
|
@classmethod
|
|
87
87
|
def get_gcp_environments(cls) -> List["Environment"]:
|
|
@@ -90,7 +90,7 @@ class Environment(Enum):
|
|
|
90
90
|
Returns:
|
|
91
91
|
List of Environment enums: [PREVIEW, GCP, LOCALHOST]
|
|
92
92
|
"""
|
|
93
|
-
return [cls.PREVIEW, cls.GCP, cls.LOCALHOST]
|
|
93
|
+
return [cls.PREVIEW, cls.GCP, cls.LOCALHOST, cls.PRODUCTION]
|
|
94
94
|
|
|
95
95
|
@classmethod
|
|
96
96
|
def get_aws_environments(cls) -> List["Environment"]:
|
|
@@ -99,4 +99,4 @@ class Environment(Enum):
|
|
|
99
99
|
Returns:
|
|
100
100
|
List of Environment enums: [PREVIEW, AWS, LOCALHOST]
|
|
101
101
|
"""
|
|
102
|
-
return [cls.PREVIEW, cls.AWS, cls.LOCALHOST]
|
|
102
|
+
return [cls.PREVIEW, cls.AWS, cls.LOCALHOST, cls.PRODUCTION]
|
|
@@ -33,6 +33,7 @@ class ModelTypes(str, Enum):
|
|
|
33
33
|
CLAUDE_35_SONNET_V2 = "claude-3-5-sonnet-v2"
|
|
34
34
|
CLAUDE_37_SONNET_V1 = "claude-3-7"
|
|
35
35
|
CLAUDE_4_SONNET = "claude-4-sonnet"
|
|
36
|
+
CLAUDE_4_5_SONNET = "claude-4-5-sonnet"
|
|
36
37
|
CLAUDE_4_OPUS = "claude-4-opus"
|
|
37
38
|
CLAUDE_4_1_OPUS = "claude-4-1-opus"
|
|
38
39
|
CLAUDE_4_SONNET_1M = "claude-4-sonnet-1m"
|
|
@@ -37,15 +37,9 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
|
|
|
37
37
|
from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
|
|
38
38
|
rp_test_data,
|
|
39
39
|
)
|
|
40
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
40
41
|
from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
|
|
41
|
-
from codemie_test_harness.tests.enums.environment import Environment
|
|
42
42
|
from codemie_test_harness.tests.utils.constants import test_project_name
|
|
43
|
-
from codemie_test_harness.tests.utils.env_resolver import get_environment
|
|
44
|
-
|
|
45
|
-
pytestmark = pytest.mark.skipif(
|
|
46
|
-
get_environment() in [Environment.LOCALHOST, Environment.GCP],
|
|
47
|
-
reason="Skipping this test on local environment",
|
|
48
|
-
)
|
|
49
43
|
|
|
50
44
|
|
|
51
45
|
@pytest.mark.assistant
|
|
@@ -87,7 +81,11 @@ def test_assistant_with_default_integration_cloud(
|
|
|
87
81
|
# create an assistant
|
|
88
82
|
cloud_assistant = assistant(toolkit, tool_name, project_name=test_project_name)
|
|
89
83
|
|
|
90
|
-
response = assistant_utils.ask_assistant(
|
|
84
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
85
|
+
cloud_assistant, prompt, minimal_response=False
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
91
89
|
|
|
92
90
|
similarity_check.check_similarity(response, expected_response)
|
|
93
91
|
|
|
@@ -126,7 +124,11 @@ def test_assistant_with_default_integration_ado(
|
|
|
126
124
|
# create an assistant
|
|
127
125
|
ado_assistant = assistant(toolkit, tool_name, project_name=test_project_name)
|
|
128
126
|
|
|
129
|
-
response = assistant_utils.ask_assistant(
|
|
127
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
128
|
+
ado_assistant, prompt, minimal_response=False
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
130
132
|
|
|
131
133
|
similarity_check.check_similarity(response, expected_response)
|
|
132
134
|
|
|
@@ -170,7 +172,11 @@ def test_assistant_with_default_integration_codebase(
|
|
|
170
172
|
toolkit, CodeBaseTool.SONAR, project_name=test_project_name
|
|
171
173
|
)
|
|
172
174
|
|
|
173
|
-
response = assistant_utils.ask_assistant(
|
|
175
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
176
|
+
sonar_assistant, prompt, minimal_response=False
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
|
|
174
180
|
|
|
175
181
|
similarity_check.check_similarity(response, expected_response)
|
|
176
182
|
|
|
@@ -223,7 +229,11 @@ def test_assistant_with_default_integration_git(
|
|
|
223
229
|
project_name=test_project_name,
|
|
224
230
|
)
|
|
225
231
|
|
|
226
|
-
response = assistant_utils.ask_assistant(
|
|
232
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
233
|
+
git_assistant, prompt, minimal_response=False
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
227
237
|
|
|
228
238
|
similarity_check.check_similarity(response, expected_response)
|
|
229
239
|
|
|
@@ -262,7 +272,11 @@ def test_assistant_with_default_integration_jira(
|
|
|
262
272
|
project_name=test_project_name,
|
|
263
273
|
)
|
|
264
274
|
|
|
265
|
-
response = assistant_utils.ask_assistant(
|
|
275
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
276
|
+
jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
|
|
266
280
|
|
|
267
281
|
similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
|
|
268
282
|
|
|
@@ -299,7 +313,11 @@ def test_assistant_with_default_integration_email(
|
|
|
299
313
|
Toolkit.NOTIFICATION, NotificationTool.EMAIL, project_name=test_project_name
|
|
300
314
|
)
|
|
301
315
|
|
|
302
|
-
response = assistant_utils.ask_assistant(
|
|
316
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
317
|
+
email_assistant, EMAIL_TOOL_PROMPT, minimal_response=False
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
assert_tool_triggered(NotificationTool.EMAIL, triggered_tools)
|
|
303
321
|
|
|
304
322
|
similarity_check.check_similarity(response, EMAIL_RESPONSE)
|
|
305
323
|
|
|
@@ -336,7 +354,11 @@ def test_assistant_with_default_integration_keycloak(
|
|
|
336
354
|
project_name=test_project_name,
|
|
337
355
|
)
|
|
338
356
|
|
|
339
|
-
response = assistant_utils.ask_assistant(
|
|
357
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
358
|
+
keycloak_assistant, KEYCLOAK_TOOL_PROMPT, minimal_response=False
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
|
|
340
362
|
|
|
341
363
|
similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
|
|
342
364
|
|
|
@@ -376,6 +398,10 @@ def test_assistant_with_default_integration_report_portal(
|
|
|
376
398
|
project_name=test_project_name,
|
|
377
399
|
)
|
|
378
400
|
|
|
379
|
-
response = assistant_utils.ask_assistant(
|
|
401
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
402
|
+
report_portal_assistant, prompt, minimal_response=False
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
assert_tool_triggered(ReportPortalTool.GET_DASHBOARD_DATA, triggered_tools)
|
|
380
406
|
|
|
381
407
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -13,10 +13,12 @@ from codemie_test_harness.tests.test_data.integrations_test_data import (
|
|
|
13
13
|
valid_integrations,
|
|
14
14
|
invalid_integrations,
|
|
15
15
|
testable_integrations,
|
|
16
|
+
empty_credentials_integrations,
|
|
16
17
|
)
|
|
17
18
|
from codemie_test_harness.tests.utils.base_utils import (
|
|
18
19
|
get_random_name,
|
|
19
20
|
assert_error_details,
|
|
21
|
+
assert_tool_triggered,
|
|
20
22
|
)
|
|
21
23
|
|
|
22
24
|
|
|
@@ -250,3 +252,43 @@ def test_delete_integration(
|
|
|
250
252
|
equal_to("Specified credential removed"),
|
|
251
253
|
"Integration delete response is not as expected.",
|
|
252
254
|
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@pytest.mark.assistant
|
|
258
|
+
@pytest.mark.integration
|
|
259
|
+
@pytest.mark.api
|
|
260
|
+
@pytest.mark.parametrize(
|
|
261
|
+
"empty_credentials, credential_type, toolkit, tool_name, prompt, expected_response",
|
|
262
|
+
empty_credentials_integrations,
|
|
263
|
+
)
|
|
264
|
+
def test_assistant_with_empty_credentials(
|
|
265
|
+
assistant_utils,
|
|
266
|
+
assistant,
|
|
267
|
+
integration,
|
|
268
|
+
similarity_check,
|
|
269
|
+
empty_credentials,
|
|
270
|
+
credential_type,
|
|
271
|
+
toolkit,
|
|
272
|
+
tool_name,
|
|
273
|
+
prompt,
|
|
274
|
+
expected_response,
|
|
275
|
+
):
|
|
276
|
+
# Create integration with empty credentials
|
|
277
|
+
empty_integration = integration(
|
|
278
|
+
credential_type=credential_type,
|
|
279
|
+
credential_values=empty_credentials,
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
assistant_instance = assistant(
|
|
283
|
+
toolkit,
|
|
284
|
+
tool_name,
|
|
285
|
+
settings=empty_integration,
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
response, triggered_tools = assistant_utils.ask_assistant(
|
|
289
|
+
assistant_instance, prompt, minimal_response=False, extract_failed_tools=True
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
293
|
+
|
|
294
|
+
similarity_check.check_similarity(response, expected_response)
|
|
@@ -38,14 +38,7 @@ from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
|
|
|
38
38
|
rp_test_data,
|
|
39
39
|
)
|
|
40
40
|
from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
|
|
41
|
-
from codemie_test_harness.tests.enums.environment import Environment
|
|
42
41
|
from codemie_test_harness.tests.utils.constants import test_project_name
|
|
43
|
-
from codemie_test_harness.tests.utils.env_resolver import get_environment
|
|
44
|
-
|
|
45
|
-
pytestmark = pytest.mark.skipif(
|
|
46
|
-
get_environment() in [Environment.LOCALHOST, Environment.GCP],
|
|
47
|
-
reason="Skipping this test on local environment",
|
|
48
|
-
)
|
|
49
42
|
|
|
50
43
|
|
|
51
44
|
@pytest.mark.assistant
|