codemie-test-harness 0.1.147__tar.gz → 0.1.148__tar.gz

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.147
+Version: 0.1.148
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.147)
+Requires-Dist: codemie-sdk-python (==0.1.148)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)

codemie_test_harness-0.1.148/codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py

@@ -0,0 +1,25 @@
+import pytest
+
+from codemie_test_harness.tests.enums.tools import Toolkit, AccessManagementTool
+from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
+    KEYCLOAK_TOOL_PROMPT,
+    KEYCLOAK_TOOL_RESPONSE,
+)
+
+
+@pytest.mark.regression
+def test_assistant_with_keycloak_tool(
+    assistant_utils,
+    assistant,
+    keycloak_integration,
+    similarity_check,
+):
+    keycloak_assistant = assistant(
+        Toolkit.ACCESS_MANAGEMENT,
+        AccessManagementTool.KEYCLOAK,
+        settings=keycloak_integration,
+    )
+
+    response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
+
+    similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/conftest.py

@@ -379,6 +379,17 @@ def service_now_integration(integration_utils):
         integration_utils.delete_integration(integration)
 
 
+@pytest.fixture(scope="module")
+def keycloak_integration(integration_utils):
+    credential_values = CredentialsUtil.keycloak_credentials()
+    integration = integration_utils.create_integration(
+        CredentialTypes.KEYCLOAK, credential_values
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
 @pytest.fixture(scope="function")
 def general_integration(integration_utils):
     created_integration: Optional[Integration] = None

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/enums/tools.py

@@ -19,6 +19,7 @@ class Toolkit(str, Enum):
     OPEN_API = "OpenAPI"
     DATA_MANAGEMENT = "Data Management"
     SERVICENOW = "IT Service Management"
+    ACCESS_MANAGEMENT = "Access Management"
 
 
 class GitTool(str, Enum):
@@ -178,3 +179,9 @@ class ServiceNowTool(str, Enum):
     """Enum for ServiceNow tool names."""
 
     SERVICE_NOW = "servicenow_table_tool"
+
+
+class AccessManagementTool(str, Enum):
+    """Enum for Access Management tool names."""
+
+    KEYCLOAK = "keycloak"

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/integrations/project/test_default_integrations.py

@@ -12,6 +12,7 @@ from codemie_test_harness.tests.enums.tools import (
     ProjectManagementTool,
     CodeBaseTool,
     CloudTool,
+    AccessManagementTool,
 )
 from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
     ado_test_plan_get_test_data,
@@ -23,6 +24,10 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
 from codemie_test_harness.tests.test_data.git_tools_test_data import (
     list_branches_set_active_branch_test_data,
 )
+from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
+    KEYCLOAK_TOOL_PROMPT,
+    KEYCLOAK_TOOL_RESPONSE,
+)
 from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_TOOL_PROMPT,
     EMAIL_RESPONSE,
@@ -270,3 +275,35 @@ def test_assistant_with_default_integration_email(
     response = assistant_utils.ask_assistant(email_assistant, EMAIL_TOOL_PROMPT)
 
     similarity_check.check_similarity(response, EMAIL_RESPONSE)
+
+
+@pytest.mark.regression
+def test_assistant_with_default_integration_keycloak(
+    general_integration,
+    integration_utils,
+    assistant,
+    assistant_utils,
+    similarity_check,
+):
+    #  delete existing integrations of the same type
+    for integration_type in IntegrationType:
+        integration_utils.delete_integrations_by_type(
+            integration_type, CredentialTypes.KEYCLOAK, test_project_name
+        )
+    # create a new integration
+    general_integration(
+        integration_type=IntegrationType.PROJECT,
+        credential_type=CredentialTypes.KEYCLOAK,
+        credential_values=CredentialsUtil.keycloak_credentials(),
+        project_name=test_project_name,
+    )
+    # create an assistant
+    keycloak_assistant = assistant(
+        Toolkit.ACCESS_MANAGEMENT,
+        AccessManagementTool.KEYCLOAK,
+        project_name=test_project_name,
+    )
+
+    response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
+
+    similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/integrations/project/test_project_integrations.py

@@ -46,6 +46,7 @@ from codemie_test_harness.tests.utils.base_utils import (
         f"Project integration: {CredentialTypes.TELEGRAM}",
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
+        f"Project integration: {CredentialTypes.KEYCLOAK}",
     ],
 )
 @pytest.mark.regression
@@ -148,6 +149,7 @@ def test_integration_after_creation(
         f"Project integration: {CredentialTypes.TELEGRAM}",
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
+        f"Project integration: {CredentialTypes.KEYCLOAK}",
     ],
 )
 @pytest.mark.tescase("EPMCDME-2376")
@@ -333,6 +335,7 @@ def test_integration_during_creation_with_invalid_credentials(
         f"Project integration: {CredentialTypes.TELEGRAM}",
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
+        f"Project integration: {CredentialTypes.KEYCLOAK}",
     ],
 )
 @pytest.mark.tescase("EPMCDME-2374")

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/integrations/user/test_default_integrations.py

@@ -12,6 +12,7 @@ from codemie_test_harness.tests.enums.tools import (
     ProjectManagementTool,
     CodeBaseTool,
     CloudTool,
+    AccessManagementTool,
 )
 from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
     ado_test_plan_get_test_data,
@@ -23,6 +24,10 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
 from codemie_test_harness.tests.test_data.git_tools_test_data import (
     list_branches_set_active_branch_test_data,
 )
+from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
+    KEYCLOAK_TOOL_PROMPT,
+    KEYCLOAK_TOOL_RESPONSE,
+)
 from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_TOOL_PROMPT,
     EMAIL_RESPONSE,
@@ -270,3 +275,35 @@ def test_assistant_with_default_integration_email(
     response = assistant_utils.ask_assistant(email_assistant, EMAIL_TOOL_PROMPT)
 
     similarity_check.check_similarity(response, EMAIL_RESPONSE)
+
+
+@pytest.mark.regression
+def test_assistant_with_default_integration_keycloak(
+    general_integration,
+    integration_utils,
+    assistant,
+    assistant_utils,
+    similarity_check,
+):
+    #  delete existing integrations of the same type
+    for integration_type in IntegrationType:
+        integration_utils.delete_integrations_by_type(
+            integration_type, CredentialTypes.KEYCLOAK, test_project_name
+        )
+    # create a new integration
+    general_integration(
+        integration_type=IntegrationType.USER,
+        credential_type=CredentialTypes.KEYCLOAK,
+        credential_values=CredentialsUtil.keycloak_credentials(),
+        project_name=test_project_name,
+    )
+    # create an assistant
+    keycloak_assistant = assistant(
+        Toolkit.ACCESS_MANAGEMENT,
+        AccessManagementTool.KEYCLOAK,
+        project_name=test_project_name,
+    )
+
+    response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
+
+    similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/integrations/user/test_user_integrations.py

@@ -46,6 +46,7 @@ from codemie_test_harness.tests.utils.base_utils import (
         f"User integration: {CredentialTypes.TELEGRAM}",
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
+        f"User integration: {CredentialTypes.KEYCLOAK}",
     ],
 )
 @pytest.mark.regression
@@ -150,6 +151,7 @@ def test_integration_after_creation(
         f"User integration: {CredentialTypes.TELEGRAM}",
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
+        f"User integration: {CredentialTypes.KEYCLOAK}",
     ],
 )
 @pytest.mark.tescase("EPMCDME-2375")
@@ -336,6 +338,7 @@ def test_integration_during_creation_with_invalid_credentials(
         f"User integration: {CredentialTypes.TELEGRAM}",
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
+        f"User integration: {CredentialTypes.KEYCLOAK}",
     ],
 )
 @pytest.mark.tescase("EPMCDME-2373")

codemie_test_harness-0.1.148/codemie_test_harness/tests/test_data/direct_tools/keycloak_tool_test_data.py

@@ -0,0 +1,120 @@
+KEYCLOAK_DIRECT_TOOL_PROMPT = {
+    "method": "GET",
+    "relative_url": "/users/profile",
+    "params": "",
+}
+
+KEYCLOAK_DIRECT_TOOL_RESPONSE = """
+    {
+    "attributes": [
+        {
+            "name": "username",
+            "displayName": "${username}",
+            "validations": {
+                "length": {
+                    "min": 3,
+                    "max": 255
+                },
+                "username-prohibited-characters": {},
+                "up-username-not-idn-homograph": {}
+            },
+            "permissions": {
+                "view": [
+                    "admin",
+                    "user"
+                ],
+                "edit": [
+                    "admin",
+                    "user"
+                ]
+            },
+            "multivalued": false
+        },
+        {
+            "name": "email",
+            "displayName": "${email}",
+            "validations": {
+                "email": {},
+                "length": {
+                    "max": 255
+                }
+            },
+            "required": {
+                "roles": [
+                    "user"
+                ]
+            },
+            "permissions": {
+                "view": [
+                    "admin",
+                    "user"
+                ],
+                "edit": [
+                    "admin",
+                    "user"
+                ]
+            },
+            "multivalued": false
+        },
+        {
+            "name": "firstName",
+            "displayName": "${firstName}",
+            "validations": {
+                "length": {
+                    "max": 255
+                },
+                "person-name-prohibited-characters": {}
+            },
+            "required": {
+                "roles": [
+                    "user"
+                ]
+            },
+            "permissions": {
+                "view": [
+                    "admin",
+                    "user"
+                ],
+                "edit": [
+                    "admin",
+                    "user"
+                ]
+            },
+            "multivalued": false
+        },
+        {
+            "name": "lastName",
+            "displayName": "${lastName}",
+            "validations": {
+                "length": {
+                    "max": 255
+                },
+                "person-name-prohibited-characters": {}
+            },
+            "required": {
+                "roles": [
+                    "user"
+                ]
+            },
+            "permissions": {
+                "view": [
+                    "admin",
+                    "user"
+                ],
+                "edit": [
+                    "admin",
+                    "user"
+                ]
+            },
+            "multivalued": false
+        }
+    ],
+    "groups": [
+        {
+            "name": "user-metadata",
+            "displayHeader": "User metadata",
+            "displayDescription": "Attributes, which refer to user metadata"
+        }
+    ]
+}
+"""

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/test_data/integrations_test_data.py

@@ -91,6 +91,10 @@ valid_integrations = [
         CredentialTypes.SERVICE_NOW,
         CredentialsUtil.servicenow_credentials(),
     ),
+    (
+        CredentialTypes.KEYCLOAK,
+        CredentialsUtil.keycloak_credentials(),
+    ),
     pytest.param(
         CredentialTypes.KUBERNETES,
         CredentialsUtil.kubernetes_credentials(),

codemie_test_harness-0.1.148/codemie_test_harness/tests/test_data/keycloak_tool_test_data.py

@@ -0,0 +1,35 @@
+KEYCLOAK_TOOL_PROMPT = "Get information about all users"
+
+KEYCLOAK_TOOL_RESPONSE = """
+        1. **User 1:**
+          - **ID:** 8246cdb1-a480-487c-9d6a-6873331dcffe
+          - **Username:** codemie-autotests-1@epam.com
+          - **First Name:** Codemie
+          - **Last Name:** Autotests-1
+          - **Email:** codemie-autotests-1@epam.com
+          - **Email Verified:** false
+          - **Created Timestamp:** 1756980669356
+          - **Enabled:** true
+          - **Access Permissions:**
+            - Manage Group Membership: true
+            - View: true
+            - Map Roles: true
+            - Impersonate: true
+            - Manage: true
+       
+       2. **User 2:**
+          - **ID:** e5710bb7-e3a5-4910-bd64-e9dcb75b1bfa
+          - **Username:** codemie-autotests-2@epam.com
+          - **First Name:** Codemie
+          - **Last Name:** Autotests-2
+          - **Email:** codemie-autotests-2@epam.com
+          - **Email Verified:** false
+          - **Created Timestamp:** 1756980685292
+          - **Enabled:** true
+          - **Access Permissions:**
+            - Manage Group Membership: true
+            - View: true
+            - Map Roles: true
+            - Impersonate: true
+            - Manage: true
+"""

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/codemie_test_harness/tests/utils/aws_parameters_store.py

@@ -564,6 +564,34 @@ class CredentialsUtil:
                 cred.value = "wrong_token"
         return credentials
 
+    @staticmethod
+    def keycloak_credentials() -> List[CredentialValues]:
+        keycloak_creds = AwsParameterStore.get_cloud_provider_credentials("keycloak")
+        return [
+            CredentialValues(
+                key="url", value=keycloak_creds.get("keycloak_admin", {}).get("url")
+            ),
+            CredentialValues(
+                key="realm", value=keycloak_creds.get("keycloak_admin", {}).get("realm")
+            ),
+            CredentialValues(
+                key="client_id",
+                value=keycloak_creds.get("keycloak_admin", {}).get("client_id"),
+            ),
+            CredentialValues(
+                key="client_secret",
+                value=keycloak_creds.get("keycloak_admin", {}).get("client_secret"),
+            ),
+        ]
+
+    @staticmethod
+    def invalid_keycloak_credentials() -> List[CredentialValues]:
+        credentials = CredentialsUtil.keycloak_credentials()
+        for cred in credentials:
+            if cred.key == "client_secret":
+                cred.value = "wrong_secret"
+        return credentials
+
     @staticmethod
     def jira_cloud_jql() -> str:
         jira_creds = AwsParameterStore.get_cloud_provider_credentials("jira")

codemie_test_harness-0.1.148/codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py

@@ -0,0 +1,28 @@
+import pytest
+
+from codemie_test_harness.tests.enums.tools import Toolkit, AccessManagementTool
+from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
+    KEYCLOAK_TOOL_PROMPT,
+    KEYCLOAK_TOOL_RESPONSE,
+)
+
+
+@pytest.mark.regression
+def test_workflow_with_assistant_with_keycloak_tool(
+    assistant,
+    workflow_with_assistant,
+    workflow_utils,
+    keycloak_integration,
+    similarity_check,
+):
+    assistant = assistant(
+        Toolkit.ACCESS_MANAGEMENT,
+        AccessManagementTool.KEYCLOAK,
+        settings=keycloak_integration,
+    )
+
+    workflow_with_assistant = workflow_with_assistant(assistant, KEYCLOAK_TOOL_PROMPT)
+    response = workflow_utils.execute_workflow(
+        workflow_with_assistant.id, assistant.name
+    )
+    similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)

codemie_test_harness-0.1.148/codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_access_management_tool.py

@@ -0,0 +1,74 @@
+import copy
+import json
+import random
+
+import pytest
+
+from codemie_test_harness.tests.enums.tools import AccessManagementTool
+from codemie_test_harness.tests.test_data.direct_tools.keycloak_tool_test_data import (
+    KEYCLOAK_DIRECT_TOOL_PROMPT,
+    KEYCLOAK_DIRECT_TOOL_RESPONSE,
+)
+from codemie_test_harness.tests.utils.base_utils import get_random_name
+
+
+@pytest.mark.regression
+@pytest.mark.skip(reason="Need to investigate codemie-tools for fix")
+def test_workflow_with_keycloak_tool_direct(
+    keycloak_integration, workflow_utils, workflow_with_tool, similarity_check
+):
+    assistant_and_state_name = get_random_name()
+
+    test_workflow = workflow_with_tool(
+        assistant_and_state_name,
+        AccessManagementTool.KEYCLOAK,
+        integration=keycloak_integration,
+    )
+    response = workflow_utils.execute_workflow(
+        test_workflow.id,
+        assistant_and_state_name,
+        user_input=json.dumps(KEYCLOAK_DIRECT_TOOL_PROMPT),
+    )
+    similarity_check.check_similarity(response, KEYCLOAK_DIRECT_TOOL_RESPONSE)
+
+
+@pytest.mark.regression
+def test_workflow_with_keycloak_tool_with_hardcoded_args(
+    keycloak_integration, workflow_utils, workflow_with_tool, similarity_check
+):
+    assistant_and_state_name = get_random_name()
+
+    test_workflow = workflow_with_tool(
+        assistant_and_state_name,
+        AccessManagementTool.KEYCLOAK,
+        integration=keycloak_integration,
+        tool_args=KEYCLOAK_DIRECT_TOOL_PROMPT,
+    )
+    response = workflow_utils.execute_workflow(
+        test_workflow.id, assistant_and_state_name
+    )
+    similarity_check.check_similarity(response, KEYCLOAK_DIRECT_TOOL_RESPONSE)
+
+
+@pytest.mark.regression
+@pytest.mark.skip(reason="Need to investigate codemie-tools for fix")
+def test_workflow_with_keycloak_tool_with_overriding_args(
+    keycloak_integration, workflow_utils, workflow_with_tool, similarity_check
+):
+    assistant_and_state_name = get_random_name()
+
+    args_copy = copy.deepcopy(KEYCLOAK_DIRECT_TOOL_PROMPT)
+    args_copy = {key: random.randint(1, 10) for key in args_copy}
+
+    test_workflow = workflow_with_tool(
+        assistant_and_state_name,
+        AccessManagementTool.KEYCLOAK,
+        integration=keycloak_integration,
+        tool_args=args_copy,
+    )
+    response = workflow_utils.execute_workflow(
+        test_workflow.id,
+        assistant_and_state_name,
+        user_input=json.dumps(KEYCLOAK_DIRECT_TOOL_PROMPT),
+    )
+    similarity_check.check_similarity(response, KEYCLOAK_DIRECT_TOOL_RESPONSE)

codemie_test_harness-0.1.148/codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py

@@ -0,0 +1,29 @@
+import pytest
+
+from codemie_test_harness.tests.enums.tools import AccessManagementTool
+from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
+    KEYCLOAK_TOOL_PROMPT,
+    KEYCLOAK_TOOL_RESPONSE,
+)
+from codemie_test_harness.tests.utils.base_utils import get_random_name
+
+
+@pytest.mark.regression
+def test_workflow_with_virtual_assistant_with_keycloak_tool(
+    keycloak_integration,
+    workflow_with_virtual_assistant,
+    workflow_utils,
+    similarity_check,
+):
+    assistant_and_state_name = get_random_name()
+
+    test_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        AccessManagementTool.KEYCLOAK,
+        integration=keycloak_integration,
+        task=KEYCLOAK_TOOL_PROMPT,
+    )
+    response = workflow_utils.execute_workflow(
+        test_workflow.id, assistant_and_state_name
+    )
+    similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)

{codemie_test_harness-0.1.147 → codemie_test_harness-0.1.148}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemie-test-harness"
-version = "0.1.147"
+version = "0.1.148"
 description = "Autotest for CodeMie backend and UI"
 authors = [
     "Anton Yeromin <anton_yeromin@epam.com>",
@@ -12,7 +12,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.12"
-codemie-sdk-python = "0.1.147"
+codemie-sdk-python = "0.1.148"
 pytest = "^8.4.1"
 python-dotenv = "^1.1.0"
 PyHamcrest = "^2.1.0"