codemie-test-harness 0.1.224__py3-none-any.whl → 0.1.226__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of codemie-test-harness might be problematic. Click here for more details.
- codemie_test_harness/tests/conftest.py +13 -1
- codemie_test_harness/tests/test_data/vendor_knowledge_base_test_data.py +84 -0
- codemie_test_harness/tests/test_data/vendor_workflow_test_data.py +83 -0
- codemie_test_harness/tests/utils/{vendor_utils.py → vendor_assistant_utils.py} +81 -2
- codemie_test_harness/tests/utils/vendor_knowledge_base_utils.py +350 -0
- codemie_test_harness/tests/utils/vendor_workflow_utils.py +1 -1
- codemie_test_harness/tests/vendor/assistants/test_vendor_assistants.py +1 -1
- codemie_test_harness/tests/vendor/knowledge_bases/__init__.py +1 -0
- codemie_test_harness/tests/vendor/knowledge_bases/test_vendor_knowledge_base_endpoints.py +285 -0
- codemie_test_harness/tests/vendor/knowledge_bases/test_vendor_knowledge_bases.py +123 -0
- codemie_test_harness/tests/vendor/workflows/test_vendor_workflows.py +1 -1
- {codemie_test_harness-0.1.224.dist-info → codemie_test_harness-0.1.226.dist-info}/METADATA +2 -2
- {codemie_test_harness-0.1.224.dist-info → codemie_test_harness-0.1.226.dist-info}/RECORD +16 -11
- /codemie_test_harness/tests/test_data/{vendor_test_data.py → vendor_assistant_test_data.py} +0 -0
- {codemie_test_harness-0.1.224.dist-info → codemie_test_harness-0.1.226.dist-info}/WHEEL +0 -0
- {codemie_test_harness-0.1.224.dist-info → codemie_test_harness-0.1.226.dist-info}/entry_points.txt +0 -0
|
@@ -953,7 +953,9 @@ def integration(integration_utils):
|
|
|
953
953
|
@pytest.fixture(scope="session")
|
|
954
954
|
def vendor_assistant_utils():
|
|
955
955
|
"""Create VendorUtils instance for managing vendor assistants"""
|
|
956
|
-
from codemie_test_harness.tests.utils.
|
|
956
|
+
from codemie_test_harness.tests.utils.vendor_assistant_utils import (
|
|
957
|
+
VendorAssistantUtils,
|
|
958
|
+
)
|
|
957
959
|
|
|
958
960
|
return VendorAssistantUtils()
|
|
959
961
|
|
|
@@ -968,6 +970,16 @@ def vendor_workflow_utils():
|
|
|
968
970
|
return VendorWorkflowUtils()
|
|
969
971
|
|
|
970
972
|
|
|
973
|
+
@pytest.fixture(scope="session")
|
|
974
|
+
def vendor_knowledge_base_utils():
|
|
975
|
+
"""Create VendorKnowledgeBaseUtils instance for managing vendor knowledge bases"""
|
|
976
|
+
from codemie_test_harness.tests.utils.vendor_knowledge_base_utils import (
|
|
977
|
+
VendorKnowledgeBaseUtils,
|
|
978
|
+
)
|
|
979
|
+
|
|
980
|
+
return VendorKnowledgeBaseUtils()
|
|
981
|
+
|
|
982
|
+
|
|
971
983
|
def pytest_sessionfinish(session):
|
|
972
984
|
"""Run cleanup code after all tests have finished."""
|
|
973
985
|
clean_up_timeout = 1 if EnvironmentResolver.is_production() else 0
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Test data for vendor knowledge base integration tests."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from codemie_sdk.models.vendor_assistant import VendorType
|
|
5
|
+
from codemie_sdk.models.integration import CredentialTypes
|
|
6
|
+
|
|
7
|
+
from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# Test data for vendor knowledge base integration tests
|
|
11
|
+
# Extensible for AWS Bedrock, Azure AI, and GCP Vertex AI
|
|
12
|
+
vendor_knowledge_base_test_data = [
|
|
13
|
+
pytest.param(
|
|
14
|
+
VendorType.AWS,
|
|
15
|
+
CredentialTypes.AWS,
|
|
16
|
+
CredentialsManager.aws_credentials(),
|
|
17
|
+
"Tell me about filtering by language",
|
|
18
|
+
"Filtering by language allows you to select books based on your preferred language. "
|
|
19
|
+
"According to the knowledge base, the available language filter options include:\n\n"
|
|
20
|
+
"- English\n- Spanish\n- French\n- German\n\n"
|
|
21
|
+
"You can use these filters to view only the books that are available in the selected language.",
|
|
22
|
+
marks=[pytest.mark.aws, pytest.mark.bedrock],
|
|
23
|
+
id="AWS_Bedrock",
|
|
24
|
+
),
|
|
25
|
+
# Future: Azure AI knowledge bases
|
|
26
|
+
# pytest.param(
|
|
27
|
+
# VendorType.AZURE,
|
|
28
|
+
# CredentialTypes.AZURE,
|
|
29
|
+
# CredentialsManager.azure_credentials(),
|
|
30
|
+
# "Tell me about filtering by language",
|
|
31
|
+
# "Filtering by language allows you to select books based on your preferred language.",
|
|
32
|
+
# marks=[pytest.mark.azure],
|
|
33
|
+
# id="Azure_AI",
|
|
34
|
+
# ),
|
|
35
|
+
# Future: GCP Vertex AI knowledge bases
|
|
36
|
+
# pytest.param(
|
|
37
|
+
# VendorType.GCP,
|
|
38
|
+
# CredentialTypes.GCP,
|
|
39
|
+
# CredentialsManager.gcp_credentials(),
|
|
40
|
+
# "Tell me about filtering by language",
|
|
41
|
+
# "Filtering by language allows you to select books based on your preferred language.",
|
|
42
|
+
# marks=[pytest.mark.gcp],
|
|
43
|
+
# id="GCP_Vertex_AI",
|
|
44
|
+
# ),
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
# Test data for vendor assistant with KB integration tests
|
|
48
|
+
vendor_assistant_with_kb_test_data = [
|
|
49
|
+
pytest.param(
|
|
50
|
+
VendorType.AWS,
|
|
51
|
+
CredentialTypes.AWS,
|
|
52
|
+
CredentialsManager.aws_credentials(),
|
|
53
|
+
"codemie-autotests-with-kb",
|
|
54
|
+
"Tell me about filtering by language",
|
|
55
|
+
"Filtering by language allows you to select books based on your preferred language. "
|
|
56
|
+
"According to the knowledge base, the available language filter options include:\n\n"
|
|
57
|
+
"- English\n- Spanish\n- French\n- German\n\n"
|
|
58
|
+
"You can use these filters to view only the books that are available in the selected language.",
|
|
59
|
+
marks=[pytest.mark.aws, pytest.mark.bedrock],
|
|
60
|
+
id="AWS_Bedrock",
|
|
61
|
+
),
|
|
62
|
+
# Future: Azure AI assistants with KB
|
|
63
|
+
# pytest.param(
|
|
64
|
+
# VendorType.AZURE,
|
|
65
|
+
# CredentialTypes.AZURE,
|
|
66
|
+
# CredentialsManager.azure_credentials(),
|
|
67
|
+
# "codemie-autotests-with-kb",
|
|
68
|
+
# "Tell me about filtering by language",
|
|
69
|
+
# "Filtering by language allows you to select books based on your preferred language.",
|
|
70
|
+
# marks=[pytest.mark.azure],
|
|
71
|
+
# id="Azure_AI",
|
|
72
|
+
# ),
|
|
73
|
+
# Future: GCP Vertex AI assistants with KB
|
|
74
|
+
# pytest.param(
|
|
75
|
+
# VendorType.GCP,
|
|
76
|
+
# CredentialTypes.GCP,
|
|
77
|
+
# CredentialsManager.gcp_credentials(),
|
|
78
|
+
# "codemie-autotests-with-kb",
|
|
79
|
+
# "Tell me about filtering by language",
|
|
80
|
+
# "Filtering by language allows you to select books based on your preferred language.",
|
|
81
|
+
# marks=[pytest.mark.gcp],
|
|
82
|
+
# id="GCP_Vertex_AI",
|
|
83
|
+
# ),
|
|
84
|
+
]
|
|
@@ -27,6 +27,89 @@ vendor_workflow_test_data = [
|
|
|
27
27
|
marks=[pytest.mark.aws, pytest.mark.bedrock],
|
|
28
28
|
id="AWS_Bedrock_Object_Input",
|
|
29
29
|
),
|
|
30
|
+
pytest.param(
|
|
31
|
+
VendorType.AWS,
|
|
32
|
+
CredentialTypes.AWS,
|
|
33
|
+
CredentialsManager.aws_credentials(),
|
|
34
|
+
"codemie-autotests-flow-string-input",
|
|
35
|
+
"pop",
|
|
36
|
+
"""
|
|
37
|
+
# Pop Playlist (3 Songs)
|
|
38
|
+
|
|
39
|
+
1. "Blinding Lights" - The Weeknd
|
|
40
|
+
*Upbeat synth-pop track with irresistible 80s vibes and a catchy chorus*
|
|
41
|
+
|
|
42
|
+
2. "As It Was" - Harry Styles
|
|
43
|
+
*Melodic pop anthem with nostalgic undertones and danceable rhythm*
|
|
44
|
+
|
|
45
|
+
3. "Levitating" - Dua Lipa
|
|
46
|
+
*Disco-infused pop banger with infectious grooves and stellar vocals*
|
|
47
|
+
|
|
48
|
+
Enjoy these modern pop hits! Would you like any specific recommendations or adjustments to the playlist?
|
|
49
|
+
""",
|
|
50
|
+
marks=[pytest.mark.aws, pytest.mark.bedrock],
|
|
51
|
+
id="AWS_Bedrock_String_Input",
|
|
52
|
+
),
|
|
53
|
+
pytest.param(
|
|
54
|
+
VendorType.AWS,
|
|
55
|
+
CredentialTypes.AWS,
|
|
56
|
+
CredentialsManager.aws_credentials(),
|
|
57
|
+
"codemie-autotests-flow-number-input",
|
|
58
|
+
json.dumps(3),
|
|
59
|
+
"""
|
|
60
|
+
# Pop Playlist (3 Songs)
|
|
61
|
+
|
|
62
|
+
1. "Shape of You" - Ed Sheeran
|
|
63
|
+
2. "Blinding Lights" - The Weeknd
|
|
64
|
+
3. "Dance Monkey" - Tones and I
|
|
65
|
+
|
|
66
|
+
Each of these songs has been a massive pop hit with catchy hooks and radio-friendly production. Enjoy your playlist!
|
|
67
|
+
""",
|
|
68
|
+
marks=[pytest.mark.aws, pytest.mark.bedrock],
|
|
69
|
+
id="AWS_Bedrock_Number_Input",
|
|
70
|
+
),
|
|
71
|
+
pytest.param(
|
|
72
|
+
VendorType.AWS,
|
|
73
|
+
CredentialTypes.AWS,
|
|
74
|
+
CredentialsManager.aws_credentials(),
|
|
75
|
+
"codemie-autotests-flow-boolean-input",
|
|
76
|
+
json.dumps(True),
|
|
77
|
+
"""
|
|
78
|
+
# Pop Playlist (3 Songs)
|
|
79
|
+
|
|
80
|
+
1. "Bad Guy" by Billie Eilish
|
|
81
|
+
2. "Blinding Lights" by The Weeknd
|
|
82
|
+
3. "As It Was" by Harry Styles
|
|
83
|
+
|
|
84
|
+
These are three popular pop songs from recent years that feature catchy melodies, modern production, and have achieved significant commercial success.
|
|
85
|
+
""",
|
|
86
|
+
marks=[pytest.mark.aws, pytest.mark.bedrock],
|
|
87
|
+
id="AWS_Bedrock_Boolean_Input",
|
|
88
|
+
),
|
|
89
|
+
pytest.param(
|
|
90
|
+
VendorType.AWS,
|
|
91
|
+
CredentialTypes.AWS,
|
|
92
|
+
CredentialsManager.aws_credentials(),
|
|
93
|
+
"codemie-autotests-flow-array-input",
|
|
94
|
+
json.dumps(["pop", "rock"]),
|
|
95
|
+
"""
|
|
96
|
+
# Playlist Creation
|
|
97
|
+
|
|
98
|
+
Here are two playlists with 2 songs each:
|
|
99
|
+
|
|
100
|
+
## Pop Playlist (2 songs)
|
|
101
|
+
1. "Shape of You" by Ed Sheeran
|
|
102
|
+
2. "Bad Guy" by Billie Eilish
|
|
103
|
+
|
|
104
|
+
## Rock Playlist (2 songs)
|
|
105
|
+
1. "Bohemian Rhapsody" by Queen
|
|
106
|
+
2. "Sweet Child O' Mine" by Guns N' Roses
|
|
107
|
+
|
|
108
|
+
Would you like me to expand either playlist with more song suggestions or provide information about any of these tracks?
|
|
109
|
+
""",
|
|
110
|
+
marks=[pytest.mark.aws, pytest.mark.bedrock],
|
|
111
|
+
id="AWS_Bedrock_Array_Input",
|
|
112
|
+
),
|
|
30
113
|
# pytest.param(
|
|
31
114
|
# VendorType.AZURE,
|
|
32
115
|
# CredentialTypes.AZURE,
|
|
@@ -161,7 +161,7 @@ class VendorAssistantUtils:
|
|
|
161
161
|
assistants: List of assistant installation requests
|
|
162
162
|
|
|
163
163
|
Returns:
|
|
164
|
-
VendorAssistantInstallResponse containing installation summary with
|
|
164
|
+
VendorAssistantInstallResponse containing installation summary with CodeMie IDs
|
|
165
165
|
"""
|
|
166
166
|
logger.info(f"Installing {len(assistants)} assistant(s) for {vendor.value}")
|
|
167
167
|
response = self.vendor_assistant_service.install_assistants(
|
|
@@ -231,6 +231,48 @@ class VendorAssistantUtils:
|
|
|
231
231
|
aliases[0] if aliases else None,
|
|
232
232
|
)
|
|
233
233
|
|
|
234
|
+
def find_assistant_by_name(
|
|
235
|
+
self,
|
|
236
|
+
vendor: VendorType,
|
|
237
|
+
setting_id: str,
|
|
238
|
+
assistant_name: str,
|
|
239
|
+
) -> Optional[tuple[VendorAssistant, str]]:
|
|
240
|
+
"""Find an assistant by name and return it with an alias.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
vendor: Cloud vendor type
|
|
244
|
+
setting_id: ID of the vendor setting
|
|
245
|
+
assistant_name: Name of the assistant to find
|
|
246
|
+
|
|
247
|
+
Returns:
|
|
248
|
+
Tuple of (VendorAssistant, alias_id) or None if assistant not found
|
|
249
|
+
"""
|
|
250
|
+
logger.info(
|
|
251
|
+
f"Searching for assistant '{assistant_name}' in {vendor.value} setting {setting_id}"
|
|
252
|
+
)
|
|
253
|
+
assistants_response = self.get_assistants(vendor=vendor, setting_id=setting_id)
|
|
254
|
+
|
|
255
|
+
for assistant in assistants_response.data:
|
|
256
|
+
if (
|
|
257
|
+
assistant.name == assistant_name
|
|
258
|
+
and assistant.status == VendorAssistantStatus.PREPARED
|
|
259
|
+
):
|
|
260
|
+
aliases_response = self.get_assistant_aliases(
|
|
261
|
+
vendor=vendor, assistant_id=assistant.id, setting_id=setting_id
|
|
262
|
+
)
|
|
263
|
+
if aliases_response.data:
|
|
264
|
+
non_draft_alias = self.get_non_draft_alias(aliases_response.data)
|
|
265
|
+
if non_draft_alias:
|
|
266
|
+
logger.info(
|
|
267
|
+
f"Found assistant: {assistant.name} (ID: {assistant.id}, Alias: {non_draft_alias.id})"
|
|
268
|
+
)
|
|
269
|
+
return assistant, non_draft_alias.id
|
|
270
|
+
|
|
271
|
+
logger.warning(
|
|
272
|
+
f"Assistant '{assistant_name}' not found for {vendor.value} setting {setting_id}"
|
|
273
|
+
)
|
|
274
|
+
return None
|
|
275
|
+
|
|
234
276
|
def find_first_available_assistant(
|
|
235
277
|
self,
|
|
236
278
|
vendor: VendorType,
|
|
@@ -252,7 +294,6 @@ class VendorAssistantUtils:
|
|
|
252
294
|
|
|
253
295
|
for assistant in assistants_response.data:
|
|
254
296
|
if assistant.status.value == "PREPARED":
|
|
255
|
-
# Get aliases for this assistant
|
|
256
297
|
aliases_response = self.get_assistant_aliases(
|
|
257
298
|
vendor=vendor, assistant_id=assistant.id, setting_id=setting_id
|
|
258
299
|
)
|
|
@@ -268,6 +309,44 @@ class VendorAssistantUtils:
|
|
|
268
309
|
)
|
|
269
310
|
return None
|
|
270
311
|
|
|
312
|
+
def install_assistant_by_name(
|
|
313
|
+
self,
|
|
314
|
+
vendor: VendorType,
|
|
315
|
+
setting_id: str,
|
|
316
|
+
assistant_name: str,
|
|
317
|
+
) -> Optional[str]:
|
|
318
|
+
"""Find and install an assistant by name.
|
|
319
|
+
|
|
320
|
+
Args:
|
|
321
|
+
vendor: Cloud vendor type
|
|
322
|
+
setting_id: ID of the vendor setting
|
|
323
|
+
assistant_name: Name of the assistant to install
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
CodeMie ID of the installed assistant or None if assistant not found
|
|
327
|
+
"""
|
|
328
|
+
result = self.find_assistant_by_name(
|
|
329
|
+
vendor=vendor, setting_id=setting_id, assistant_name=assistant_name
|
|
330
|
+
)
|
|
331
|
+
if not result:
|
|
332
|
+
return None
|
|
333
|
+
|
|
334
|
+
assistant, alias_id = result
|
|
335
|
+
install_request = VendorAssistantInstallRequest(
|
|
336
|
+
id=assistant.id,
|
|
337
|
+
agentAliasId=alias_id,
|
|
338
|
+
setting_id=setting_id,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
install_response = self.install_assistants(
|
|
342
|
+
vendor=vendor, assistants=[install_request]
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
if install_response.summary:
|
|
346
|
+
return install_response.summary[0].aiRunId
|
|
347
|
+
|
|
348
|
+
return None
|
|
349
|
+
|
|
271
350
|
def install_first_available_assistant(
|
|
272
351
|
self,
|
|
273
352
|
vendor: VendorType,
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
"""Utility class for managing vendor knowledge bases (AWS Bedrock, Azure, GCP)."""
|
|
2
|
+
|
|
3
|
+
from typing import Optional, List
|
|
4
|
+
from codemie_sdk.models.vendor_assistant import VendorType
|
|
5
|
+
from codemie_sdk.models.vendor_knowledgebase import (
|
|
6
|
+
VendorKnowledgeBaseSettingsResponse,
|
|
7
|
+
VendorKnowledgeBasesResponse,
|
|
8
|
+
VendorKnowledgeBase,
|
|
9
|
+
VendorKnowledgeBaseDetail,
|
|
10
|
+
VendorKnowledgeBaseInstallRequest,
|
|
11
|
+
VendorKnowledgeBaseInstallResponse,
|
|
12
|
+
VendorKnowledgeBaseUninstallResponse,
|
|
13
|
+
VendorKnowledgeBaseStatus,
|
|
14
|
+
)
|
|
15
|
+
from codemie_test_harness.tests.utils.client_factory import get_client
|
|
16
|
+
from codemie_test_harness.tests.utils.logger_util import setup_logger
|
|
17
|
+
|
|
18
|
+
logger = setup_logger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class VendorKnowledgeBaseUtils:
|
|
22
|
+
"""Utility class for vendor knowledge base operations."""
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
"""Initialize VendorKnowledgeBaseUtils with CodeMie client."""
|
|
26
|
+
self.client = get_client()
|
|
27
|
+
self.vendor_kb_service = self.client.vendor_knowledgebases
|
|
28
|
+
|
|
29
|
+
def get_knowledgebase_settings(
|
|
30
|
+
self,
|
|
31
|
+
vendor: VendorType,
|
|
32
|
+
page: int = 0,
|
|
33
|
+
per_page: int = 10,
|
|
34
|
+
) -> VendorKnowledgeBaseSettingsResponse:
|
|
35
|
+
"""Get knowledge base settings for a specific cloud vendor.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
vendor: Cloud vendor type (AWS, AZURE, GCP)
|
|
39
|
+
page: Page number for pagination
|
|
40
|
+
per_page: Number of items per page
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
VendorKnowledgeBaseSettingsResponse containing list of settings
|
|
44
|
+
"""
|
|
45
|
+
logger.info(
|
|
46
|
+
f"Getting KB settings for {vendor.value} (page={page}, per_page={per_page})"
|
|
47
|
+
)
|
|
48
|
+
settings = self.vendor_kb_service.get_knowledgebase_settings(
|
|
49
|
+
vendor=vendor, page=page, per_page=per_page
|
|
50
|
+
)
|
|
51
|
+
logger.info(
|
|
52
|
+
f"Retrieved {len(settings.data)} KB settings for {vendor.value} (total: {settings.pagination.total})"
|
|
53
|
+
)
|
|
54
|
+
return settings
|
|
55
|
+
|
|
56
|
+
def get_knowledgebases(
|
|
57
|
+
self,
|
|
58
|
+
vendor: VendorType,
|
|
59
|
+
setting_id: str,
|
|
60
|
+
per_page: int = 10,
|
|
61
|
+
next_token: Optional[str] = None,
|
|
62
|
+
) -> VendorKnowledgeBasesResponse:
|
|
63
|
+
"""Get knowledge bases for a specific vendor setting.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
vendor: Cloud vendor type
|
|
67
|
+
setting_id: ID of the vendor setting
|
|
68
|
+
per_page: Number of items per page
|
|
69
|
+
next_token: Token for pagination
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
VendorKnowledgeBasesResponse containing list of knowledge bases
|
|
73
|
+
"""
|
|
74
|
+
logger.info(
|
|
75
|
+
f"Getting knowledge bases for {vendor.value} setting {setting_id} (per_page={per_page})"
|
|
76
|
+
)
|
|
77
|
+
kbs = self.vendor_kb_service.get_knowledgebases(
|
|
78
|
+
vendor=vendor,
|
|
79
|
+
setting_id=setting_id,
|
|
80
|
+
per_page=per_page,
|
|
81
|
+
next_token=next_token,
|
|
82
|
+
)
|
|
83
|
+
logger.info(
|
|
84
|
+
f"Retrieved {len(kbs.data)} knowledge bases for setting {setting_id}"
|
|
85
|
+
)
|
|
86
|
+
return kbs
|
|
87
|
+
|
|
88
|
+
def get_knowledgebase(
|
|
89
|
+
self,
|
|
90
|
+
vendor: VendorType,
|
|
91
|
+
knowledgebase_id: str,
|
|
92
|
+
setting_id: str,
|
|
93
|
+
) -> VendorKnowledgeBaseDetail:
|
|
94
|
+
"""Get a specific knowledge base by ID.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
vendor: Cloud vendor type
|
|
98
|
+
knowledgebase_id: ID of the knowledge base
|
|
99
|
+
setting_id: ID of the vendor setting
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
VendorKnowledgeBaseDetail with knowledge base details
|
|
103
|
+
"""
|
|
104
|
+
logger.info(
|
|
105
|
+
f"Getting knowledge base {knowledgebase_id} for {vendor.value} setting {setting_id}"
|
|
106
|
+
)
|
|
107
|
+
kb = self.vendor_kb_service.get_knowledgebase(
|
|
108
|
+
vendor=vendor, knowledgebase_id=knowledgebase_id, setting_id=setting_id
|
|
109
|
+
)
|
|
110
|
+
logger.info(f"Retrieved knowledge base: {kb.name} (status: {kb.status})")
|
|
111
|
+
return kb
|
|
112
|
+
|
|
113
|
+
def install_knowledgebases(
|
|
114
|
+
self,
|
|
115
|
+
vendor: VendorType,
|
|
116
|
+
knowledgebases: List[VendorKnowledgeBaseInstallRequest],
|
|
117
|
+
) -> VendorKnowledgeBaseInstallResponse:
|
|
118
|
+
"""Install/activate vendor knowledge bases.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
vendor: Cloud vendor type
|
|
122
|
+
knowledgebases: List of knowledge base installation requests
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
VendorKnowledgeBaseInstallResponse containing installation summary
|
|
126
|
+
"""
|
|
127
|
+
logger.info(
|
|
128
|
+
f"Installing {len(knowledgebases)} knowledge base(s) for {vendor.value}"
|
|
129
|
+
)
|
|
130
|
+
response = self.vendor_kb_service.install_knowledgebases(
|
|
131
|
+
vendor=vendor, knowledgebases=knowledgebases
|
|
132
|
+
)
|
|
133
|
+
for item in response.summary:
|
|
134
|
+
logger.info(
|
|
135
|
+
f"Installed KB {item.knowledgeBaseId} -> CodeMie ID: {item.aiRunId}"
|
|
136
|
+
)
|
|
137
|
+
return response
|
|
138
|
+
|
|
139
|
+
def uninstall_knowledgebase(
|
|
140
|
+
self,
|
|
141
|
+
vendor: VendorType,
|
|
142
|
+
codemie_id: str,
|
|
143
|
+
) -> VendorKnowledgeBaseUninstallResponse:
|
|
144
|
+
"""Uninstall/deactivate a vendor knowledge base.
|
|
145
|
+
|
|
146
|
+
Args:
|
|
147
|
+
vendor: Cloud vendor type
|
|
148
|
+
codemie_id: CodeMie ID from installation
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
VendorKnowledgeBaseUninstallResponse with success status
|
|
152
|
+
"""
|
|
153
|
+
logger.info(f"Uninstalling knowledge base with CodeMie ID: {codemie_id}")
|
|
154
|
+
response = self.vendor_kb_service.uninstall_knowledgebase(
|
|
155
|
+
vendor=vendor, ai_run_id=codemie_id
|
|
156
|
+
)
|
|
157
|
+
if response.success:
|
|
158
|
+
logger.info(f"Successfully uninstalled knowledge base {codemie_id}")
|
|
159
|
+
else:
|
|
160
|
+
logger.warning(f"Failed to uninstall knowledge base {codemie_id}")
|
|
161
|
+
return response
|
|
162
|
+
|
|
163
|
+
def get_available_knowledge_base(
|
|
164
|
+
self,
|
|
165
|
+
knowledge_bases: List[VendorKnowledgeBase],
|
|
166
|
+
) -> Optional[VendorKnowledgeBase]:
|
|
167
|
+
"""Get first PREPARED knowledge base from the list.
|
|
168
|
+
|
|
169
|
+
Args:
|
|
170
|
+
knowledge_bases: List of vendor knowledge bases
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
First PREPARED knowledge base or first KB if none are PREPARED
|
|
174
|
+
"""
|
|
175
|
+
return next(
|
|
176
|
+
(
|
|
177
|
+
kb
|
|
178
|
+
for kb in knowledge_bases
|
|
179
|
+
if kb.status == VendorKnowledgeBaseStatus.PREPARED
|
|
180
|
+
),
|
|
181
|
+
knowledge_bases[0] if knowledge_bases else None,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
def find_setting_for_integration(
|
|
185
|
+
self,
|
|
186
|
+
vendor: VendorType,
|
|
187
|
+
integration_id: str,
|
|
188
|
+
):
|
|
189
|
+
"""Find setting for an integration by paginating through all settings.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
vendor: Type of vendor (AWS, AZURE, GCP)
|
|
193
|
+
integration_id: ID of the integration to find (searches by setting_id)
|
|
194
|
+
|
|
195
|
+
Returns:
|
|
196
|
+
VendorKnowledgeBaseSetting or None if not found
|
|
197
|
+
"""
|
|
198
|
+
page = 0
|
|
199
|
+
per_page = 50
|
|
200
|
+
|
|
201
|
+
while True:
|
|
202
|
+
settings_response = self.get_knowledgebase_settings(
|
|
203
|
+
vendor=vendor,
|
|
204
|
+
page=page,
|
|
205
|
+
per_page=per_page,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
# Find the setting for our integration by setting_id
|
|
209
|
+
for s in settings_response.data:
|
|
210
|
+
if s.setting_id == integration_id:
|
|
211
|
+
return s
|
|
212
|
+
|
|
213
|
+
# Check if there are more pages
|
|
214
|
+
if page >= settings_response.pagination.pages - 1:
|
|
215
|
+
break
|
|
216
|
+
|
|
217
|
+
page += 1
|
|
218
|
+
|
|
219
|
+
logger.warning(f"Setting not found for integration ID '{integration_id}'")
|
|
220
|
+
return None
|
|
221
|
+
|
|
222
|
+
def find_knowledge_base_by_name(
|
|
223
|
+
self,
|
|
224
|
+
vendor: VendorType,
|
|
225
|
+
setting_id: str,
|
|
226
|
+
kb_name: str,
|
|
227
|
+
) -> Optional[VendorKnowledgeBase]:
|
|
228
|
+
"""Find a knowledge base by name.
|
|
229
|
+
|
|
230
|
+
Args:
|
|
231
|
+
vendor: Cloud vendor type
|
|
232
|
+
setting_id: ID of the vendor setting
|
|
233
|
+
kb_name: Name of the knowledge base to find
|
|
234
|
+
|
|
235
|
+
Returns:
|
|
236
|
+
VendorKnowledgeBase or None if knowledge base not found
|
|
237
|
+
"""
|
|
238
|
+
logger.info(
|
|
239
|
+
f"Searching for knowledge base '{kb_name}' in {vendor.value} setting {setting_id}"
|
|
240
|
+
)
|
|
241
|
+
kbs_response = self.get_knowledgebases(vendor=vendor, setting_id=setting_id)
|
|
242
|
+
|
|
243
|
+
for kb in kbs_response.data:
|
|
244
|
+
if kb.name == kb_name and kb.status == VendorKnowledgeBaseStatus.PREPARED:
|
|
245
|
+
logger.info(f"Found knowledge base: {kb.name} (ID: {kb.id})")
|
|
246
|
+
return kb
|
|
247
|
+
|
|
248
|
+
logger.warning(
|
|
249
|
+
f"Knowledge base '{kb_name}' not found for {vendor.value} setting {setting_id}"
|
|
250
|
+
)
|
|
251
|
+
return None
|
|
252
|
+
|
|
253
|
+
def find_first_available_knowledge_base(
|
|
254
|
+
self,
|
|
255
|
+
vendor: VendorType,
|
|
256
|
+
setting_id: str,
|
|
257
|
+
) -> Optional[VendorKnowledgeBase]:
|
|
258
|
+
"""Find the first prepared knowledge base.
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
vendor: Cloud vendor type
|
|
262
|
+
setting_id: ID of the vendor setting
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
VendorKnowledgeBase or None if no prepared knowledge base found
|
|
266
|
+
"""
|
|
267
|
+
logger.info(
|
|
268
|
+
f"Searching for prepared knowledge base in {vendor.value} setting {setting_id}"
|
|
269
|
+
)
|
|
270
|
+
kbs_response = self.get_knowledgebases(vendor=vendor, setting_id=setting_id)
|
|
271
|
+
|
|
272
|
+
for kb in kbs_response.data:
|
|
273
|
+
if kb.status == VendorKnowledgeBaseStatus.PREPARED:
|
|
274
|
+
logger.info(f"Found prepared knowledge base: {kb.name} (ID: {kb.id})")
|
|
275
|
+
return kb
|
|
276
|
+
|
|
277
|
+
logger.warning(
|
|
278
|
+
f"No prepared knowledge base found for {vendor.value} setting {setting_id}"
|
|
279
|
+
)
|
|
280
|
+
return None
|
|
281
|
+
|
|
282
|
+
def install_knowledge_base_by_name(
|
|
283
|
+
self,
|
|
284
|
+
vendor: VendorType,
|
|
285
|
+
setting_id: str,
|
|
286
|
+
kb_name: str,
|
|
287
|
+
) -> Optional[str]:
|
|
288
|
+
"""Find and install a knowledge base by name.
|
|
289
|
+
|
|
290
|
+
Args:
|
|
291
|
+
vendor: Cloud vendor type
|
|
292
|
+
setting_id: ID of the vendor setting
|
|
293
|
+
kb_name: Name of the knowledge base to install
|
|
294
|
+
|
|
295
|
+
Returns:
|
|
296
|
+
CodeMie ID of the installed knowledge base or None if not found
|
|
297
|
+
"""
|
|
298
|
+
kb = self.find_knowledge_base_by_name(
|
|
299
|
+
vendor=vendor, setting_id=setting_id, kb_name=kb_name
|
|
300
|
+
)
|
|
301
|
+
if not kb:
|
|
302
|
+
return None
|
|
303
|
+
|
|
304
|
+
install_request = VendorKnowledgeBaseInstallRequest(
|
|
305
|
+
id=kb.id,
|
|
306
|
+
setting_id=setting_id,
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
install_response = self.install_knowledgebases(
|
|
310
|
+
vendor=vendor, knowledgebases=[install_request]
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
if install_response.summary:
|
|
314
|
+
return install_response.summary[0].aiRunId
|
|
315
|
+
|
|
316
|
+
return None
|
|
317
|
+
|
|
318
|
+
def install_first_available_knowledge_base(
|
|
319
|
+
self,
|
|
320
|
+
vendor: VendorType,
|
|
321
|
+
setting_id: str,
|
|
322
|
+
) -> Optional[str]:
|
|
323
|
+
"""Find and install the first available knowledge base.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
vendor: Cloud vendor type
|
|
327
|
+
setting_id: ID of the vendor setting
|
|
328
|
+
|
|
329
|
+
Returns:
|
|
330
|
+
CodeMie ID of the installed knowledge base or None if not available
|
|
331
|
+
"""
|
|
332
|
+
kb = self.find_first_available_knowledge_base(
|
|
333
|
+
vendor=vendor, setting_id=setting_id
|
|
334
|
+
)
|
|
335
|
+
if not kb:
|
|
336
|
+
return None
|
|
337
|
+
|
|
338
|
+
install_request = VendorKnowledgeBaseInstallRequest(
|
|
339
|
+
id=kb.id,
|
|
340
|
+
setting_id=setting_id,
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
install_response = self.install_knowledgebases(
|
|
344
|
+
vendor=vendor, knowledgebases=[install_request]
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
if install_response.summary:
|
|
348
|
+
return install_response.summary[0].aiRunId
|
|
349
|
+
|
|
350
|
+
return None
|
|
@@ -157,7 +157,7 @@ class VendorWorkflowUtils:
|
|
|
157
157
|
workflows: List of workflow installation requests
|
|
158
158
|
|
|
159
159
|
Returns:
|
|
160
|
-
VendorWorkflowInstallResponse containing installation summary with
|
|
160
|
+
VendorWorkflowInstallResponse containing installation summary with CodeMie IDs
|
|
161
161
|
"""
|
|
162
162
|
logger.info(f"Installing {len(workflows)} workflow(s) for {vendor.value}")
|
|
163
163
|
response = self.vendor_workflow_service.install_workflows(
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Tests for vendor knowledge base endpoints and functionality."""
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Tests for vendor knowledge base SDK endpoints.
|
|
2
|
+
|
|
3
|
+
Architecture:
|
|
4
|
+
- Parametrized tests that support multiple vendors (AWS, Azure, GCP)
|
|
5
|
+
- Extensible design for easy addition of new vendors
|
|
6
|
+
- Uses integration fixture factory for consistent setup
|
|
7
|
+
- Each test validates a specific SDK endpoint
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
from hamcrest import (
|
|
12
|
+
assert_that,
|
|
13
|
+
is_not,
|
|
14
|
+
none,
|
|
15
|
+
empty,
|
|
16
|
+
greater_than,
|
|
17
|
+
instance_of,
|
|
18
|
+
has_length,
|
|
19
|
+
is_,
|
|
20
|
+
)
|
|
21
|
+
from codemie_sdk.models.vendor_knowledgebase import (
|
|
22
|
+
VendorKnowledgeBaseSettingsResponse,
|
|
23
|
+
VendorKnowledgeBasesResponse,
|
|
24
|
+
VendorKnowledgeBaseDetail,
|
|
25
|
+
VendorKnowledgeBaseInstallRequest,
|
|
26
|
+
VendorKnowledgeBaseInstallResponse,
|
|
27
|
+
VendorKnowledgeBaseUninstallResponse,
|
|
28
|
+
VendorKnowledgeBaseStatus,
|
|
29
|
+
)
|
|
30
|
+
from codemie_test_harness.tests.test_data.vendor_endpoint_test_data import (
|
|
31
|
+
vendor_endpoint_test_data,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@pytest.mark.vendor
|
|
36
|
+
@pytest.mark.api
|
|
37
|
+
@pytest.mark.parametrize(
|
|
38
|
+
"vendor_type,credential_type,credentials",
|
|
39
|
+
vendor_endpoint_test_data,
|
|
40
|
+
)
|
|
41
|
+
def test_get_knowledgebase_settings(
|
|
42
|
+
vendor_knowledge_base_utils, integration, vendor_type, credential_type, credentials
|
|
43
|
+
):
|
|
44
|
+
"""Test GET /v1/vendors/{vendor}/knowledge-bases/settings endpoint."""
|
|
45
|
+
_integration = integration(credential_type, credentials)
|
|
46
|
+
|
|
47
|
+
settings_response = vendor_knowledge_base_utils.get_knowledgebase_settings(
|
|
48
|
+
vendor=vendor_type
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
assert_that(settings_response, instance_of(VendorKnowledgeBaseSettingsResponse))
|
|
52
|
+
assert_that(settings_response.data, is_not(none()))
|
|
53
|
+
assert_that(settings_response.pagination, is_not(none()))
|
|
54
|
+
assert_that(settings_response.data, is_not(empty()))
|
|
55
|
+
assert_that(settings_response.pagination.total, greater_than(0))
|
|
56
|
+
|
|
57
|
+
created_setting = vendor_knowledge_base_utils.find_setting_for_integration(
|
|
58
|
+
vendor=vendor_type,
|
|
59
|
+
integration_id=_integration.id,
|
|
60
|
+
)
|
|
61
|
+
assert_that(created_setting, is_not(none()))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@pytest.mark.vendor
|
|
65
|
+
@pytest.mark.api
|
|
66
|
+
@pytest.mark.parametrize(
|
|
67
|
+
"vendor_type,credential_type,credentials",
|
|
68
|
+
vendor_endpoint_test_data,
|
|
69
|
+
)
|
|
70
|
+
def test_get_knowledgebases(
|
|
71
|
+
vendor_knowledge_base_utils, integration, vendor_type, credential_type, credentials
|
|
72
|
+
):
|
|
73
|
+
"""Test GET /v1/vendors/{vendor}/knowledge-bases endpoint."""
|
|
74
|
+
_integration = integration(credential_type, credentials)
|
|
75
|
+
|
|
76
|
+
setting = vendor_knowledge_base_utils.find_setting_for_integration(
|
|
77
|
+
vendor=vendor_type,
|
|
78
|
+
integration_id=_integration.id,
|
|
79
|
+
)
|
|
80
|
+
assert_that(setting, is_not(none()))
|
|
81
|
+
|
|
82
|
+
kbs_response = vendor_knowledge_base_utils.get_knowledgebases(
|
|
83
|
+
vendor=vendor_type,
|
|
84
|
+
setting_id=setting.setting_id,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
assert_that(kbs_response, instance_of(VendorKnowledgeBasesResponse))
|
|
88
|
+
assert_that(kbs_response.data, is_not(none()))
|
|
89
|
+
assert_that(kbs_response.pagination, is_not(none()))
|
|
90
|
+
assert_that(kbs_response.data, is_not(empty()))
|
|
91
|
+
|
|
92
|
+
_kb = vendor_knowledge_base_utils.get_available_knowledge_base(kbs_response.data)
|
|
93
|
+
|
|
94
|
+
assert_that(_kb.id, is_not(none()))
|
|
95
|
+
assert_that(_kb.name, is_not(none()))
|
|
96
|
+
assert_that(_kb.status, is_(VendorKnowledgeBaseStatus.PREPARED))
|
|
97
|
+
assert_that(_kb.description, is_not(none()))
|
|
98
|
+
assert_that(_kb.updatedAt, is_not(none()))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@pytest.mark.vendor
|
|
102
|
+
@pytest.mark.api
|
|
103
|
+
@pytest.mark.parametrize(
|
|
104
|
+
"vendor_type,credential_type,credentials",
|
|
105
|
+
vendor_endpoint_test_data,
|
|
106
|
+
)
|
|
107
|
+
def test_get_knowledgebase(
|
|
108
|
+
vendor_knowledge_base_utils, integration, vendor_type, credential_type, credentials
|
|
109
|
+
):
|
|
110
|
+
"""Test GET /v1/vendors/{vendor}/knowledge-bases/{knowledgebase_id} endpoint."""
|
|
111
|
+
_integration = integration(credential_type, credentials)
|
|
112
|
+
|
|
113
|
+
setting = vendor_knowledge_base_utils.find_setting_for_integration(
|
|
114
|
+
vendor=vendor_type,
|
|
115
|
+
integration_id=_integration.id,
|
|
116
|
+
)
|
|
117
|
+
assert_that(setting, is_not(none()))
|
|
118
|
+
|
|
119
|
+
kbs_response = vendor_knowledge_base_utils.get_knowledgebases(
|
|
120
|
+
vendor=vendor_type,
|
|
121
|
+
setting_id=setting.setting_id,
|
|
122
|
+
)
|
|
123
|
+
assert_that(kbs_response.data, is_not(empty()))
|
|
124
|
+
|
|
125
|
+
first_kb = vendor_knowledge_base_utils.get_available_knowledge_base(
|
|
126
|
+
kbs_response.data
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
_kb = vendor_knowledge_base_utils.get_knowledgebase(
|
|
130
|
+
vendor=vendor_type,
|
|
131
|
+
knowledgebase_id=first_kb.id,
|
|
132
|
+
setting_id=setting.setting_id,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
assert_that(_kb, instance_of(VendorKnowledgeBaseDetail))
|
|
136
|
+
assert_that(_kb.id, is_(first_kb.id))
|
|
137
|
+
assert_that(_kb.name, is_not(none()))
|
|
138
|
+
assert_that(_kb.status, is_(VendorKnowledgeBaseStatus.PREPARED))
|
|
139
|
+
assert_that(_kb.description, is_not(none()))
|
|
140
|
+
assert_that(_kb.updatedAt, is_not(none()))
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@pytest.mark.vendor
|
|
144
|
+
@pytest.mark.api
|
|
145
|
+
@pytest.mark.parametrize(
|
|
146
|
+
"vendor_type,credential_type,credentials",
|
|
147
|
+
vendor_endpoint_test_data,
|
|
148
|
+
)
|
|
149
|
+
def test_install_knowledgebases(
|
|
150
|
+
vendor_knowledge_base_utils, integration, vendor_type, credential_type, credentials
|
|
151
|
+
):
|
|
152
|
+
"""Test POST /v1/vendors/{vendor}/knowledge-bases endpoint."""
|
|
153
|
+
_integration = integration(credential_type, credentials)
|
|
154
|
+
|
|
155
|
+
setting = vendor_knowledge_base_utils.find_setting_for_integration(
|
|
156
|
+
vendor=vendor_type,
|
|
157
|
+
integration_id=_integration.id,
|
|
158
|
+
)
|
|
159
|
+
assert_that(setting, is_not(none()))
|
|
160
|
+
|
|
161
|
+
kb = vendor_knowledge_base_utils.find_first_available_knowledge_base(
|
|
162
|
+
vendor=vendor_type,
|
|
163
|
+
setting_id=setting.setting_id,
|
|
164
|
+
)
|
|
165
|
+
assert_that(kb, is_not(none()))
|
|
166
|
+
|
|
167
|
+
install_request = VendorKnowledgeBaseInstallRequest(
|
|
168
|
+
id=kb.id,
|
|
169
|
+
setting_id=setting.setting_id,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
install_response = vendor_knowledge_base_utils.install_knowledgebases(
|
|
173
|
+
vendor=vendor_type,
|
|
174
|
+
knowledgebases=[install_request],
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
assert_that(install_response, instance_of(VendorKnowledgeBaseInstallResponse))
|
|
178
|
+
assert_that(install_response.summary, is_not(empty()))
|
|
179
|
+
assert_that(install_response.summary, has_length(1))
|
|
180
|
+
|
|
181
|
+
installed = install_response.summary[0]
|
|
182
|
+
assert_that(installed.knowledgeBaseId, is_(kb.id))
|
|
183
|
+
assert_that(installed.aiRunId, is_not(none()))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@pytest.mark.vendor
|
|
187
|
+
@pytest.mark.api
|
|
188
|
+
@pytest.mark.parametrize(
|
|
189
|
+
"vendor_type,credential_type,credentials",
|
|
190
|
+
vendor_endpoint_test_data,
|
|
191
|
+
)
|
|
192
|
+
def test_uninstall_knowledgebase(
|
|
193
|
+
vendor_knowledge_base_utils, integration, vendor_type, credential_type, credentials
|
|
194
|
+
):
|
|
195
|
+
"""Test DELETE /v1/vendors/{vendor}/knowledge-bases/{ai_run_id} endpoint."""
|
|
196
|
+
_integration = integration(credential_type, credentials)
|
|
197
|
+
|
|
198
|
+
setting = vendor_knowledge_base_utils.find_setting_for_integration(
|
|
199
|
+
vendor=vendor_type,
|
|
200
|
+
integration_id=_integration.id,
|
|
201
|
+
)
|
|
202
|
+
assert_that(setting, is_not(none()))
|
|
203
|
+
|
|
204
|
+
codemie_id = vendor_knowledge_base_utils.install_first_available_knowledge_base(
|
|
205
|
+
vendor=vendor_type,
|
|
206
|
+
setting_id=setting.setting_id,
|
|
207
|
+
)
|
|
208
|
+
assert_that(codemie_id, is_not(none()))
|
|
209
|
+
|
|
210
|
+
uninstall_response = vendor_knowledge_base_utils.uninstall_knowledgebase(
|
|
211
|
+
vendor=vendor_type,
|
|
212
|
+
codemie_id=codemie_id,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
assert_that(uninstall_response, instance_of(VendorKnowledgeBaseUninstallResponse))
|
|
216
|
+
assert_that(uninstall_response.success, is_(True))
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@pytest.mark.vendor
|
|
220
|
+
@pytest.mark.api
|
|
221
|
+
@pytest.mark.parametrize(
|
|
222
|
+
"vendor_type,credential_type,credentials",
|
|
223
|
+
vendor_endpoint_test_data,
|
|
224
|
+
)
|
|
225
|
+
def test_get_knowledgebase_settings_pagination(
|
|
226
|
+
vendor_knowledge_base_utils, integration, vendor_type, credential_type, credentials
|
|
227
|
+
):
|
|
228
|
+
"""Test pagination functionality for get_knowledgebase_settings endpoint."""
|
|
229
|
+
first_page = vendor_knowledge_base_utils.get_knowledgebase_settings(
|
|
230
|
+
vendor=vendor_type,
|
|
231
|
+
page=0,
|
|
232
|
+
per_page=2,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
assert_that(first_page.pagination.page, is_(0))
|
|
236
|
+
assert_that(first_page.pagination.per_page, is_(2))
|
|
237
|
+
assert_that(first_page.pagination.total, greater_than(0))
|
|
238
|
+
|
|
239
|
+
if first_page.pagination.pages > 1:
|
|
240
|
+
second_page = vendor_knowledge_base_utils.get_knowledgebase_settings(
|
|
241
|
+
vendor=vendor_type,
|
|
242
|
+
page=1,
|
|
243
|
+
per_page=2,
|
|
244
|
+
)
|
|
245
|
+
assert_that(second_page.pagination.page, is_(1))
|
|
246
|
+
assert_that(second_page.data, is_not(empty()))
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@pytest.mark.vendor
|
|
250
|
+
@pytest.mark.api
|
|
251
|
+
@pytest.mark.parametrize(
|
|
252
|
+
"vendor_type,credential_type,credentials",
|
|
253
|
+
vendor_endpoint_test_data,
|
|
254
|
+
)
|
|
255
|
+
def test_get_knowledgebases_pagination(
|
|
256
|
+
vendor_knowledge_base_utils, integration, vendor_type, credential_type, credentials
|
|
257
|
+
):
|
|
258
|
+
"""Test pagination functionality for get_knowledgebases endpoint using next_token."""
|
|
259
|
+
_integration = integration(credential_type, credentials)
|
|
260
|
+
|
|
261
|
+
setting = vendor_knowledge_base_utils.find_setting_for_integration(
|
|
262
|
+
vendor=vendor_type,
|
|
263
|
+
integration_id=_integration.id,
|
|
264
|
+
)
|
|
265
|
+
assert_that(setting, is_not(none()))
|
|
266
|
+
|
|
267
|
+
first_page = vendor_knowledge_base_utils.get_knowledgebases(
|
|
268
|
+
vendor=vendor_type,
|
|
269
|
+
setting_id=setting.setting_id,
|
|
270
|
+
per_page=2,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
assert_that(first_page.data, is_not(empty()))
|
|
274
|
+
|
|
275
|
+
if first_page.pagination.next_token:
|
|
276
|
+
second_page = vendor_knowledge_base_utils.get_knowledgebases(
|
|
277
|
+
vendor=vendor_type,
|
|
278
|
+
setting_id=setting.setting_id,
|
|
279
|
+
per_page=2,
|
|
280
|
+
next_token=first_page.pagination.next_token,
|
|
281
|
+
)
|
|
282
|
+
assert_that(second_page.data, is_not(empty()))
|
|
283
|
+
first_page_ids = {kb.id for kb in first_page.data}
|
|
284
|
+
second_page_ids = {kb.id for kb in second_page.data}
|
|
285
|
+
assert_that(first_page_ids.intersection(second_page_ids), is_(set()))
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Integration tests for vendor knowledge bases with CodeMie assistants."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from hamcrest import assert_that, is_not, none, is_
|
|
5
|
+
|
|
6
|
+
from codemie_test_harness.tests.test_data.vendor_knowledge_base_test_data import (
|
|
7
|
+
vendor_knowledge_base_test_data,
|
|
8
|
+
vendor_assistant_with_kb_test_data,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.mark.vendor
|
|
13
|
+
@pytest.mark.api
|
|
14
|
+
@pytest.mark.parametrize(
|
|
15
|
+
"vendor_type,credential_type,credentials,prompt,expected_response",
|
|
16
|
+
vendor_knowledge_base_test_data,
|
|
17
|
+
)
|
|
18
|
+
def test_codemie_assistant_with_vendor_knowledgebase(
|
|
19
|
+
vendor_knowledge_base_utils,
|
|
20
|
+
integration,
|
|
21
|
+
assistant,
|
|
22
|
+
assistant_utils,
|
|
23
|
+
similarity_check,
|
|
24
|
+
kb_context,
|
|
25
|
+
client,
|
|
26
|
+
vendor_type,
|
|
27
|
+
credential_type,
|
|
28
|
+
credentials,
|
|
29
|
+
prompt,
|
|
30
|
+
expected_response,
|
|
31
|
+
):
|
|
32
|
+
"""Test CodeMie assistant with vendor knowledge base integration."""
|
|
33
|
+
_integration = integration(credential_type, credentials)
|
|
34
|
+
|
|
35
|
+
setting = vendor_knowledge_base_utils.find_setting_for_integration(
|
|
36
|
+
vendor=vendor_type,
|
|
37
|
+
integration_id=_integration.id,
|
|
38
|
+
)
|
|
39
|
+
assert_that(setting, is_not(none()))
|
|
40
|
+
assert_that(setting.invalid or False, is_(False))
|
|
41
|
+
|
|
42
|
+
codemie_kb_id = vendor_knowledge_base_utils.install_first_available_knowledge_base(
|
|
43
|
+
vendor=vendor_type,
|
|
44
|
+
setting_id=setting.setting_id,
|
|
45
|
+
)
|
|
46
|
+
assert_that(codemie_kb_id, is_not(none()))
|
|
47
|
+
|
|
48
|
+
datasource = client.datasources.get(codemie_kb_id)
|
|
49
|
+
assert_that(datasource, is_not(none()))
|
|
50
|
+
|
|
51
|
+
created_assistant = assistant(
|
|
52
|
+
context=kb_context(datasource),
|
|
53
|
+
system_prompt="You are a helpful assistant. Use the knowledge base to answer questions.",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
response = assistant_utils.ask_assistant(
|
|
57
|
+
assistant=created_assistant,
|
|
58
|
+
user_prompt=prompt,
|
|
59
|
+
minimal_response=True,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
similarity_check.check_similarity(response, expected_response)
|
|
63
|
+
|
|
64
|
+
uninstall_response = vendor_knowledge_base_utils.uninstall_knowledgebase(
|
|
65
|
+
vendor=vendor_type,
|
|
66
|
+
codemie_id=codemie_kb_id,
|
|
67
|
+
)
|
|
68
|
+
assert_that(uninstall_response.success, is_(True))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@pytest.mark.vendor
|
|
72
|
+
@pytest.mark.api
|
|
73
|
+
@pytest.mark.parametrize(
|
|
74
|
+
"vendor_type,credential_type,credentials,assistant_name,prompt,expected_response",
|
|
75
|
+
vendor_assistant_with_kb_test_data,
|
|
76
|
+
)
|
|
77
|
+
def test_vendor_assistant_with_knowledgebase(
|
|
78
|
+
vendor_assistant_utils,
|
|
79
|
+
integration,
|
|
80
|
+
assistant_utils,
|
|
81
|
+
similarity_check,
|
|
82
|
+
vendor_type,
|
|
83
|
+
credential_type,
|
|
84
|
+
credentials,
|
|
85
|
+
assistant_name,
|
|
86
|
+
prompt,
|
|
87
|
+
expected_response,
|
|
88
|
+
):
|
|
89
|
+
"""Test vendor assistant with knowledge base already attached."""
|
|
90
|
+
_integration = integration(credential_type, credentials)
|
|
91
|
+
|
|
92
|
+
setting = vendor_assistant_utils.find_setting_for_integration(
|
|
93
|
+
vendor=vendor_type,
|
|
94
|
+
integration_id=_integration.id,
|
|
95
|
+
)
|
|
96
|
+
assert_that(setting, is_not(none()))
|
|
97
|
+
assert_that(setting.invalid or False, is_(False))
|
|
98
|
+
|
|
99
|
+
codemie_id = vendor_assistant_utils.install_assistant_by_name(
|
|
100
|
+
vendor=vendor_type,
|
|
101
|
+
setting_id=setting.setting_id,
|
|
102
|
+
assistant_name=assistant_name,
|
|
103
|
+
)
|
|
104
|
+
assert_that(codemie_id, is_not(none()))
|
|
105
|
+
|
|
106
|
+
class AssistantIdWrapper:
|
|
107
|
+
def __init__(self, assistant_id):
|
|
108
|
+
self.id = assistant_id
|
|
109
|
+
|
|
110
|
+
assistant_wrapper = AssistantIdWrapper(codemie_id)
|
|
111
|
+
response = assistant_utils.ask_assistant(
|
|
112
|
+
assistant=assistant_wrapper,
|
|
113
|
+
user_prompt=prompt,
|
|
114
|
+
minimal_response=True,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
similarity_check.check_similarity(response, expected_response)
|
|
118
|
+
|
|
119
|
+
uninstall_response = vendor_assistant_utils.uninstall_assistant(
|
|
120
|
+
vendor=vendor_type,
|
|
121
|
+
codemie_id=codemie_id,
|
|
122
|
+
)
|
|
123
|
+
assert_that(uninstall_response.success, is_(True))
|
|
@@ -54,7 +54,7 @@ def test_vendor_workflow_installation_and_execution(
|
|
|
54
54
|
user_input=user_input,
|
|
55
55
|
)
|
|
56
56
|
|
|
57
|
-
similarity_check.check_similarity(response, expected_response)
|
|
57
|
+
similarity_check.check_similarity(response, expected_response, similarity_rank=50)
|
|
58
58
|
|
|
59
59
|
uninstall_response = vendor_workflow_utils.uninstall_workflow(
|
|
60
60
|
vendor=vendor_type,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: codemie-test-harness
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.226
|
|
4
4
|
Summary: Autotest for CodeMie backend and UI
|
|
5
5
|
Author: Anton Yeromin
|
|
6
6
|
Author-email: anton_yeromin@epam.com
|
|
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
|
|
|
13
13
|
Requires-Dist: boto3 (>=1.39.8,<2.0.0)
|
|
14
14
|
Requires-Dist: click (>=8.1.7,<9.0.0)
|
|
15
15
|
Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
|
|
16
|
-
Requires-Dist: codemie-sdk-python (==0.1.
|
|
16
|
+
Requires-Dist: codemie-sdk-python (==0.1.226)
|
|
17
17
|
Requires-Dist: pytest (>=8.4.1,<9.0.0)
|
|
18
18
|
Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
|
|
19
19
|
Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)
|
|
@@ -65,7 +65,7 @@ codemie_test_harness/tests/assistant/tools/servicenow/__init__.py,sha256=47DEQpj
|
|
|
65
65
|
codemie_test_harness/tests/assistant/tools/servicenow/test_servicenow_tools.py,sha256=aUjfZ4773WoQJjcHx3JqH5e8ckaKB-aIMO-OZWTm0Ek,888
|
|
66
66
|
codemie_test_harness/tests/assistant/tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
67
67
|
codemie_test_harness/tests/assistant/tools/vcs/test_assistant_with_vcs_tools.py,sha256=qOPr4XOh2rgUV2MXMxkRzRGkAKl9ViwQGCZ-dMEtscU,1145
|
|
68
|
-
codemie_test_harness/tests/conftest.py,sha256=
|
|
68
|
+
codemie_test_harness/tests/conftest.py,sha256=ZfngRtxFi004c831e1cFkaj0S15pIHd4nRRoqf92nS0,34805
|
|
69
69
|
codemie_test_harness/tests/conversations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
70
70
|
codemie_test_harness/tests/conversations/test_conversations_endpoints.py,sha256=HQ2nu9lXfRNkyJhA0rzar7Rmv6pMe-te0rFYAy-X5UA,4128
|
|
71
71
|
codemie_test_harness/tests/e2e/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -170,9 +170,10 @@ codemie_test_harness/tests/test_data/report_portal_tools_test_data.py,sha256=YZd
|
|
|
170
170
|
codemie_test_harness/tests/test_data/research_tools_test_data.py,sha256=zwpzm-VSnrLZEfG97AE9Ms7z7j3xmqxiNd1EmZyWCSk,9102
|
|
171
171
|
codemie_test_harness/tests/test_data/servicenow_tools_test_data.py,sha256=PKw9zEYSNcQM1KApCSjsBiA_3Py0bNQI7clqw8cmT-s,1983
|
|
172
172
|
codemie_test_harness/tests/test_data/vcs_tools_test_data.py,sha256=ZJdday96uOKOy_kPRBPp0w9JZbf0il7Y_hPfwRmUkuM,4518
|
|
173
|
+
codemie_test_harness/tests/test_data/vendor_assistant_test_data.py,sha256=FFS35ikzMzSHShnfmdIme-WjueSjj0qBwa0OU6ooKfc,1409
|
|
173
174
|
codemie_test_harness/tests/test_data/vendor_endpoint_test_data.py,sha256=Fti3ZbpS3m06Heyh5AAcoAKZuimX709iZQFPhzKvrC4,1075
|
|
174
|
-
codemie_test_harness/tests/test_data/
|
|
175
|
-
codemie_test_harness/tests/test_data/vendor_workflow_test_data.py,sha256=
|
|
175
|
+
codemie_test_harness/tests/test_data/vendor_knowledge_base_test_data.py,sha256=mWrtao3VOU5gO3_k-eiiVB4og_DtqBYkwcASPuCbysM,3401
|
|
176
|
+
codemie_test_harness/tests/test_data/vendor_workflow_test_data.py,sha256=xcemVCa2i5SR1qJMNGQ2iND1e0kizdtG4Qh1M_xy9bA,4649
|
|
176
177
|
codemie_test_harness/tests/test_data/workflow/invalid_config/invalid_assistant_id.yaml,sha256=_cioQNq3icemob9u0i-hXkTy2nflzyP0Ce8FWiPG14M,265
|
|
177
178
|
codemie_test_harness/tests/test_data/workflow/invalid_config/invalid_assistant_in_state.yaml,sha256=t_W95zD5bfdGf3F6p64-2qBHz7SkL_7mFT675uieWZg,209
|
|
178
179
|
codemie_test_harness/tests/test_data/workflow/invalid_config/invalid_data_source.yaml,sha256=Vwx3HyrQkL8sWNtfwL6d0qiJhru6X3ojKBASAzJeY9w,252
|
|
@@ -298,18 +299,22 @@ codemie_test_harness/tests/utils/pytest_utils.py,sha256=k-mEjX2qpnh37sqKpJqYhZT6
|
|
|
298
299
|
codemie_test_harness/tests/utils/search_utils.py,sha256=SrXiB2d9wiI5ka9bgg0CD73GOX_1mqi2Hz5FBm5DsEU,1435
|
|
299
300
|
codemie_test_harness/tests/utils/similarity_check.py,sha256=2URqvD3Ft7efwLmhh2iYVsXrYboP9f-_B_ekZmJn0ac,1527
|
|
300
301
|
codemie_test_harness/tests/utils/user_utils.py,sha256=zJNrmL3Fb7iGuaVRobUMwJ2Og6NqEPcM_9lw60m18T8,242
|
|
301
|
-
codemie_test_harness/tests/utils/
|
|
302
|
-
codemie_test_harness/tests/utils/
|
|
302
|
+
codemie_test_harness/tests/utils/vendor_assistant_utils.py,sha256=r8g6Vara_aUNWL1GnmcM9ESta2Qtny1B4XgWhYjY63k,13761
|
|
303
|
+
codemie_test_harness/tests/utils/vendor_knowledge_base_utils.py,sha256=UXBM1GLfbD9oqaj--jeFFcKlzG_NnElPJjJ1imQmaec,11274
|
|
304
|
+
codemie_test_harness/tests/utils/vendor_workflow_utils.py,sha256=X2v_keH94BBI6vQBgPWTl9kR8a2oy9xW0LolK-clGTY,13596
|
|
303
305
|
codemie_test_harness/tests/utils/webhook_utils.py,sha256=YjyLwAqQjR12vYFOUmYhJCJIyZvKm4SvU-1oIjIYNqg,340
|
|
304
306
|
codemie_test_harness/tests/utils/workflow_utils.py,sha256=bHxPQkblRPF1fZp_AXI36elMFm14nzqGYoU5Eqz4MWY,11553
|
|
305
307
|
codemie_test_harness/tests/utils/yaml_utils.py,sha256=iIdEl-rUUh1LgzAmD_mjfftthhvlzXyCuA37yBoH0Gw,1617
|
|
306
308
|
codemie_test_harness/tests/vendor/__init__.py,sha256=7kcktgm5ZkmcVLRX2mTRhGzVzFpVR_rcgYUbhKe-3q0,71
|
|
307
309
|
codemie_test_harness/tests/vendor/assistants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
308
|
-
codemie_test_harness/tests/vendor/assistants/test_vendor_assistants.py,sha256=
|
|
310
|
+
codemie_test_harness/tests/vendor/assistants/test_vendor_assistants.py,sha256=whgbSHDP2qjuW4gG_zo4k_aQxcMQsgfF2yqDP_IBkUE,1967
|
|
309
311
|
codemie_test_harness/tests/vendor/assistants/test_vendor_assistants_endpoints.py,sha256=XA9idTm6ZycoKCTGq31QpL_BeybgQa7dFC3RSqAPtec,13271
|
|
312
|
+
codemie_test_harness/tests/vendor/knowledge_bases/__init__.py,sha256=CiKEVPKSbGvB8UqpmISTAl6ohztOrrAdfFQpzNSToIc,67
|
|
313
|
+
codemie_test_harness/tests/vendor/knowledge_bases/test_vendor_knowledge_base_endpoints.py,sha256=GNehIqybcBAIW_jA4pYV0DrHidMz6LgHOksZ0-AaOp0,9426
|
|
314
|
+
codemie_test_harness/tests/vendor/knowledge_bases/test_vendor_knowledge_bases.py,sha256=sRejPbm9QIyQIVKBGgOxy6KcpFi888WccaWzTXRjTJc,3652
|
|
310
315
|
codemie_test_harness/tests/vendor/workflows/__init__.py,sha256=qcP6VHdlR6qbNhjrqRGHE5JReIj4CH0WGlMvpKgm1aM,38
|
|
311
316
|
codemie_test_harness/tests/vendor/workflows/test_vendor_workflow_endpoints.py,sha256=dkCmFPKj-sIVgbuaqQ2Ef7JBKUWGTftW9xZB3D1KCrU,11236
|
|
312
|
-
codemie_test_harness/tests/vendor/workflows/test_vendor_workflows.py,sha256=
|
|
317
|
+
codemie_test_harness/tests/vendor/workflows/test_vendor_workflows.py,sha256=sHMbfc6-uG0nYFtp0CBIyyGFs8DbRdux-WfD4lWsKB8,1833
|
|
313
318
|
codemie_test_harness/tests/webhook/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
314
319
|
codemie_test_harness/tests/webhook/test_webhook_service.py,sha256=POmxQG0tpcNW9-yKQ62CcnQpUEFYlTOs0_4H9MijIHY,8127
|
|
315
320
|
codemie_test_harness/tests/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -417,7 +422,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/__init__.
|
|
|
417
422
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py,sha256=D835gaRbCnB4va5mi9TdA_u9StSpGXQ_fgzwW0S2pwo,1173
|
|
418
423
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
419
424
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py,sha256=Se9imIiBYuJU78m1pLu0g4ZmHygKZjr6JjIWkGXTy1Q,1364
|
|
420
|
-
codemie_test_harness-0.1.
|
|
421
|
-
codemie_test_harness-0.1.
|
|
422
|
-
codemie_test_harness-0.1.
|
|
423
|
-
codemie_test_harness-0.1.
|
|
425
|
+
codemie_test_harness-0.1.226.dist-info/METADATA,sha256=iz3R84OYZ7pyNiD8xyzf8CyL_TrBay_sLamMH_g5Kr8,27184
|
|
426
|
+
codemie_test_harness-0.1.226.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
|
|
427
|
+
codemie_test_harness-0.1.226.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
|
|
428
|
+
codemie_test_harness-0.1.226.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{codemie_test_harness-0.1.224.dist-info → codemie_test_harness-0.1.226.dist-info}/entry_points.txt
RENAMED
|
File without changes
|