codemie-test-harness 0.1.221__tar.gz → 0.1.222__tar.gz

{codemie_test_harness-0.1.221 → codemie_test_harness-0.1.222}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.221
+Version: 0.1.222
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.221)
+Requires-Dist: codemie-sdk-python (==0.1.222)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)

{codemie_test_harness-0.1.221 → codemie_test_harness-0.1.222}/codemie_test_harness/tests/conftest.py

@@ -950,6 +950,14 @@ def integration(integration_utils):
         integration_utils.delete_integration(created_integration)
 
 
+@pytest.fixture(scope="session")
+def vendor_utils():
+    """Create VendorUtils instance for managing vendor assistants"""
+    from codemie_test_harness.tests.utils.vendor_utils import VendorUtils
+
+    return VendorUtils()
+
+
 def pytest_sessionfinish(session):
     """Run cleanup code after all tests have finished."""
     clean_up_timeout = 1 if EnvironmentResolver.is_production() else 0

codemie_test_harness-0.1.222/codemie_test_harness/tests/test_data/vendor_endpoint_test_data.py

@@ -0,0 +1,36 @@
+"""Test data for vendor service endpoint tests."""
+
+import pytest
+from codemie_sdk.models.vendor_assistant import VendorType
+from codemie_sdk.models.integration import CredentialTypes
+
+from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
+
+
+# Test data for vendor endpoint tests
+# Extensible for AWS Bedrock, Azure AI, and GCP Vertex AI
+vendor_endpoint_test_data = [
+    pytest.param(
+        VendorType.AWS,
+        CredentialTypes.AWS,
+        CredentialsManager.aws_credentials(),
+        marks=[pytest.mark.aws, pytest.mark.bedrock],
+        id="AWS_Bedrock",
+    ),
+    # Future: Azure AI endpoints
+    # pytest.param(
+    #     VendorType.AZURE,
+    #     CredentialTypes.AZURE,
+    #     CredentialsManager.azure_credentials(),
+    #     marks=[pytest.mark.azure],
+    #     id="Azure_AI",
+    # ),
+    # Future: GCP Vertex AI endpoints
+    # pytest.param(
+    #     VendorType.GCP,
+    #     CredentialTypes.GCP,
+    #     CredentialsManager.gcp_credentials(),
+    #     marks=[pytest.mark.gcp],
+    #     id="GCP_Vertex_AI",
+    # ),
+]

codemie_test_harness-0.1.222/codemie_test_harness/tests/test_data/vendor_test_data.py

@@ -0,0 +1,43 @@
+"""Test data for 3rd party vendor assistant tests."""
+
+import pytest
+from codemie_sdk.models.vendor_assistant import VendorType
+from codemie_sdk.models.integration import CredentialTypes
+
+from codemie_test_harness.tests.llm.assistants.test_llm import SIMPLE_GREETING_PROMPT
+from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
+
+
+# Test data for vendor assistant tests
+# Extensible for AWS Bedrock, Azure AI, and GCP Vertex AI
+vendor_assistant_test_data = [
+    pytest.param(
+        VendorType.AWS,
+        CredentialTypes.AWS,
+        CredentialsManager.aws_credentials(),
+        SIMPLE_GREETING_PROMPT,
+        "Hello",
+        marks=[pytest.mark.aws, pytest.mark.bedrock],
+        id="AWS_Bedrock",
+    ),
+    # Future: Azure AI assistants
+    # pytest.param(
+    #     VendorType.AZURE,
+    #     CredentialTypes.AZURE,
+    #     CredentialsManager.azure_credentials(),
+    #     "Just say one word: 'Hello'",
+    #     "Hello",
+    #     marks=[pytest.mark.vendor, pytest.mark.azure, pytest.mark.api],
+    #     id="Azure_AI",
+    # ),
+    # Future: GCP Vertex AI assistants
+    # pytest.param(
+    #     VendorType.GCP,
+    #     CredentialTypes.GCP,
+    #     CredentialsManager.gcp_credentials(),
+    #     "Just say one word: 'Hello'",
+    #     "Hello",
+    #     marks=[pytest.mark.vendor, pytest.mark.gcp, pytest.mark.api],
+    #     id="GCP_Vertex_AI",
+    # ),
+]

codemie_test_harness-0.1.222/codemie_test_harness/tests/utils/vendor_utils.py

@@ -0,0 +1,343 @@
+"""Utility class for managing vendor assistants (AWS Bedrock, Azure, GCP)."""
+
+from typing import Optional, List
+from codemie_sdk.models.vendor_assistant import (
+    VendorType,
+    VendorAssistantSettingsResponse,
+    VendorAssistantsResponse,
+    VendorAssistant,
+    VendorAssistantAliasesResponse,
+    VendorAssistantInstallRequest,
+    VendorAssistantInstallResponse,
+    VendorAssistantUninstallResponse,
+    VendorAssistantStatus,
+)
+from codemie_test_harness.tests.utils.client_factory import get_client
+from codemie_test_harness.tests.utils.logger_util import setup_logger
+
+logger = setup_logger(__name__)
+
+DRAFT_VERSION = "DRAFT"
+
+
+class VendorUtils:
+    """Utility class for vendor assistant operations."""
+
+    def __init__(self):
+        """Initialize VendorUtils with CodeMie client."""
+        self.client = get_client()
+        self.vendor_service = self.client.vendors
+
+    def get_assistant_settings(
+        self,
+        vendor: VendorType,
+        page: int = 0,
+        per_page: int = 10,
+    ) -> VendorAssistantSettingsResponse:
+        """Get assistant settings for a specific cloud vendor.
+
+        Args:
+            vendor: Cloud vendor type (AWS, AZURE, GCP)
+            page: Page number for pagination
+            per_page: Number of items per page
+
+        Returns:
+            VendorAssistantSettingsResponse containing list of settings
+        """
+        logger.info(
+            f"Getting assistant settings for {vendor.value} (page={page}, per_page={per_page})"
+        )
+        settings = self.vendor_service.get_assistant_settings(
+            vendor=vendor, page=page, per_page=per_page
+        )
+        logger.info(
+            f"Retrieved {len(settings.data)} settings for {vendor.value} (total: {settings.pagination.total})"
+        )
+        return settings
+
+    def get_assistants(
+        self,
+        vendor: VendorType,
+        setting_id: str,
+        per_page: int = 10,
+        next_token: Optional[str] = None,
+    ) -> VendorAssistantsResponse:
+        """Get assistants for a specific vendor setting.
+
+        Args:
+            vendor: Cloud vendor type
+            setting_id: ID of the vendor setting
+            per_page: Number of items per page
+            next_token: Token for pagination
+
+        Returns:
+            VendorAssistantsResponse containing list of assistants
+        """
+        logger.info(
+            f"Getting assistants for {vendor.value} setting {setting_id} (per_page={per_page})"
+        )
+        assistants = self.vendor_service.get_assistants(
+            vendor=vendor,
+            setting_id=setting_id,
+            per_page=per_page,
+            next_token=next_token,
+        )
+        logger.info(
+            f"Retrieved {len(assistants.data)} assistants for setting {setting_id}"
+        )
+        return assistants
+
+    def get_assistant(
+        self,
+        vendor: VendorType,
+        assistant_id: str,
+        setting_id: str,
+    ) -> VendorAssistant:
+        """Get a specific assistant by ID.
+
+        Args:
+            vendor: Cloud vendor type
+            assistant_id: ID of the assistant
+            setting_id: ID of the vendor setting
+
+        Returns:
+            VendorAssistant with assistant details
+        """
+        logger.info(
+            f"Getting assistant {assistant_id} for {vendor.value} setting {setting_id}"
+        )
+        assistant = self.vendor_service.get_assistant(
+            vendor=vendor, assistant_id=assistant_id, setting_id=setting_id
+        )
+        logger.info(
+            f"Retrieved assistant: {assistant.name} (status: {assistant.status})"
+        )
+        return assistant
+
+    def get_assistant_aliases(
+        self,
+        vendor: VendorType,
+        assistant_id: str,
+        setting_id: str,
+        per_page: int = 10,
+        next_token: Optional[str] = None,
+    ) -> VendorAssistantAliasesResponse:
+        """Get aliases for a specific vendor assistant.
+
+        Args:
+            vendor: Cloud vendor type
+            assistant_id: ID of the assistant
+            setting_id: ID of the vendor setting
+            per_page: Number of items per page
+            next_token: Token for pagination
+
+        Returns:
+            VendorAssistantAliasesResponse containing list of aliases
+        """
+        logger.info(
+            f"Getting aliases for assistant {assistant_id} in {vendor.value} setting {setting_id}"
+        )
+        aliases = self.vendor_service.get_assistant_aliases(
+            vendor=vendor,
+            assistant_id=assistant_id,
+            setting_id=setting_id,
+            per_page=per_page,
+            next_token=next_token,
+        )
+        logger.info(
+            f"Retrieved {len(aliases.data)} aliases for assistant {assistant_id}"
+        )
+        return aliases
+
+    def install_assistants(
+        self,
+        vendor: VendorType,
+        assistants: List[VendorAssistantInstallRequest],
+    ) -> VendorAssistantInstallResponse:
+        """Install/activate vendor assistants.
+
+        Args:
+            vendor: Cloud vendor type
+            assistants: List of assistant installation requests
+
+        Returns:
+            VendorAssistantInstallResponse containing installation summary with AI run IDs
+        """
+        logger.info(f"Installing {len(assistants)} assistant(s) for {vendor.value}")
+        response = self.vendor_service.install_assistants(
+            vendor=vendor, assistants=assistants
+        )
+        for item in response.summary:
+            logger.info(
+                f"Installed assistant {item.agentId} (alias: {item.agentAliasId}) -> CodeMie ID: {item.aiRunId}"
+            )
+        return response
+
+    def uninstall_assistant(
+        self,
+        vendor: VendorType,
+        codemie_id: str,
+    ) -> VendorAssistantUninstallResponse:
+        """Uninstall/deactivate a vendor assistant.
+
+        Args:
+            vendor: Cloud vendor type
+            codemie_id: CodeMie assistant ID from installation
+
+        Returns:
+            VendorAssistantUninstallResponse with success status
+        """
+        logger.info(f"Uninstalling assistant with CodeMie ID: {codemie_id}")
+        response = self.vendor_service.uninstall_assistant(
+            vendor=vendor, ai_run_id=codemie_id
+        )
+        if response.success:
+            logger.info(f"Successfully uninstalled assistant {codemie_id}")
+        else:
+            logger.warning(f"Failed to uninstall assistant {codemie_id}")
+        return response
+
+    def get_prepared_assistant(
+        self,
+        assistants: List[VendorAssistant],
+    ) -> Optional[VendorAssistant]:
+        """Get first PREPARED assistant from the list.
+
+        Args:
+            assistants: List of vendor assistants
+
+        Returns:
+            First PREPARED assistant or first assistant if none are PREPARED
+        """
+        return next(
+            (a for a in assistants if a.status == VendorAssistantStatus.PREPARED),
+            assistants[0] if assistants else None,
+        )
+
+    def get_non_draft_alias(
+        self,
+        aliases: List,
+    ) -> Optional:
+        """Get first non-DRAFT alias from the list.
+
+        Args:
+            aliases: List of vendor assistant aliases
+
+        Returns:
+            First non-DRAFT alias or first alias if all are DRAFT
+        """
+        return next(
+            (a for a in aliases if a.version != DRAFT_VERSION),
+            aliases[0] if aliases else None,
+        )
+
+    def find_first_available_assistant(
+        self,
+        vendor: VendorType,
+        setting_id: str,
+    ) -> Optional[tuple[VendorAssistant, str]]:
+        """Find the first available (PREPARED) assistant with an alias.
+
+        Args:
+            vendor: Cloud vendor type
+            setting_id: ID of the vendor setting
+
+        Returns:
+            Tuple of (VendorAssistant, alias_id) or None if no available assistant found
+        """
+        logger.info(
+            f"Searching for available assistant in {vendor.value} setting {setting_id}"
+        )
+        assistants_response = self.get_assistants(vendor=vendor, setting_id=setting_id)
+
+        for assistant in assistants_response.data:
+            if assistant.status.value == "PREPARED":
+                # Get aliases for this assistant
+                aliases_response = self.get_assistant_aliases(
+                    vendor=vendor, assistant_id=assistant.id, setting_id=setting_id
+                )
+                if aliases_response.data:
+                    first_alias = aliases_response.data[0]
+                    logger.info(
+                        f"Found available assistant: {assistant.name} (ID: {assistant.id}, Alias: {first_alias.id})"
+                    )
+                    return assistant, first_alias.id
+
+        logger.warning(
+            f"No available assistant found for {vendor.value} setting {setting_id}"
+        )
+        return None
+
+    def install_first_available_assistant(
+        self,
+        vendor: VendorType,
+        setting_id: str,
+    ) -> Optional[str]:
+        """Find and install the first available assistant.
+
+        Args:
+            vendor: Cloud vendor type
+            setting_id: ID of the vendor setting
+
+        Returns:
+            CodeMie ID of the installed assistant or None if no assistant available
+        """
+        result = self.find_first_available_assistant(
+            vendor=vendor, setting_id=setting_id
+        )
+        if not result:
+            return None
+
+        assistant, alias_id = result
+        install_request = VendorAssistantInstallRequest(
+            id=assistant.id,
+            agentAliasId=alias_id,
+            setting_id=setting_id,
+        )
+
+        install_response = self.install_assistants(
+            vendor=vendor, assistants=[install_request]
+        )
+
+        if install_response.summary:
+            return install_response.summary[0].aiRunId
+
+        return None
+
+    def find_setting_for_integration(
+        self,
+        vendor: VendorType,
+        integration_id: str,
+    ):
+        """Find setting for an integration by paginating through all settings.
+
+        Args:
+            vendor: Type of vendor (AWS, AZURE, GCP)
+            integration_id: ID of the integration to find (searches by setting_id)
+
+        Returns:
+            VendorAssistantSetting or None if not found
+        """
+        page = 0
+        per_page = 50
+
+        while True:
+            settings_response = self.get_assistant_settings(
+                vendor=vendor,
+                page=page,
+                per_page=per_page,
+            )
+
+            # Find the setting for our integration by setting_id
+            for s in settings_response.data:
+                if s.setting_id == integration_id:
+                    return s
+
+            # Check if there are more pages
+            if page >= settings_response.pagination.pages - 1:
+                break
+
+            page += 1
+
+        logger.warning(f"Setting not found for integration ID '{integration_id}'")
+        return None

codemie_test_harness-0.1.222/codemie_test_harness/tests/vendor/__init__.py

@@ -0,0 +1 @@
+"""Tests for 3rd party vendor assistants (AWS Bedrock, Azure, GCP)."""

codemie_test_harness-0.1.222/codemie_test_harness/tests/vendor/assistants/test_vendor_assistants.py

@@ -0,0 +1,65 @@
+import pytest
+from hamcrest import assert_that, is_not, none, empty, is_
+from codemie_test_harness.tests.test_data.vendor_test_data import (
+    vendor_assistant_test_data,
+)
+
+
+@pytest.mark.vendor
+@pytest.mark.api
+@pytest.mark.parametrize(
+    "vendor_type,credential_type,credentials,prompt,expected_response",
+    vendor_assistant_test_data,
+)
+def test_vendor_assistant_installation_and_chat(
+    vendor_utils,
+    integration,
+    assistant_utils,
+    similarity_check,
+    vendor_type,
+    credential_type,
+    credentials,
+    prompt,
+    expected_response,
+):
+    """Test vendor assistant installation and chat functionality."""
+    _integration = integration(credential_type, credentials)
+
+    setting = vendor_utils.find_setting_for_integration(
+        vendor=vendor_type,
+        integration_id=_integration.id,
+    )
+    assert_that(setting, is_not(none()))
+    assert_that(setting.invalid or False, is_(False))
+
+    assistants_response = vendor_utils.get_assistants(
+        vendor=vendor_type,
+        setting_id=setting.setting_id,
+    )
+    assert_that(assistants_response, is_not(none()))
+    assert_that(assistants_response.data, is_not(empty()))
+
+    codemie_id = vendor_utils.install_first_available_assistant(
+        vendor=vendor_type,
+        setting_id=setting.setting_id,
+    )
+    assert_that(codemie_id, is_not(none()))
+
+    class AssistantIdWrapper:
+        def __init__(self, assistant_id):
+            self.id = assistant_id
+
+    assistant_wrapper = AssistantIdWrapper(codemie_id)
+    response = assistant_utils.ask_assistant(
+        assistant=assistant_wrapper,
+        user_prompt=prompt,
+        minimal_response=True,
+    )
+
+    similarity_check.check_similarity(response, expected_response)
+
+    uninstall_response = vendor_utils.uninstall_assistant(
+        vendor=vendor_type,
+        codemie_id=codemie_id,
+    )
+    assert_that(uninstall_response.success, is_(True))