codemie-test-harness 0.1.218__py3-none-any.whl → 0.1.221__py3-none-any.whl

codemie_test_harness/tests/assistant/test_sub_assistants.py

@@ -0,0 +1,371 @@
+"""
+Tests for assistant functionality with sub-assistants.
+
+This module tests the ability to create assistants with sub-assistants,
+delegate tasks between parent and child assistants, and manage hierarchical
+assistant structures.
+"""
+
+import pytest
+from codemie_sdk.models.assistant import (
+    AssistantUpdateRequest,
+)
+from hamcrest import (
+    assert_that,
+    equal_to,
+    has_items,
+)
+
+from codemie_test_harness.tests import PROJECT
+from codemie_test_harness.tests.enums.tools import (
+    ProjectManagementTool,
+    GitTool,
+    FileManagementTool,
+    Toolkit,
+)
+from codemie_test_harness.tests.test_data.file_management_tools_test_data import (
+    CODE_INTERPRETER_TOOL_TASK,
+    RESPONSE_FOR_CODE_INTERPRETER,
+)
+from codemie_test_harness.tests.test_data.git_tools_test_data import (
+    list_branches_set_active_branch_test_data,
+)
+from codemie_test_harness.tests.test_data.pm_tools_test_data import (
+    JIRA_TOOL_PROMPT,
+    RESPONSE_FOR_JIRA_TOOL,
+)
+from codemie_test_harness.tests.utils.base_utils import assert_error_details
+
+
+@pytest.mark.assistant
+@pytest.mark.sub_assistant
+@pytest.mark.api
+@pytest.mark.smoke
+def test_create_assistant_with_single_sub_assistant(assistant, assistant_utils):
+    """
+    Test creating an assistant with a single sub-assistant.
+
+    Verifies that:
+    - Parent assistant can be created with a sub-assistant reference
+    - Sub-assistant is properly linked to parent assistant
+    - The assistant_ids field contains the sub-assistant ID
+    """
+    # Create sub-assistant first
+    sub_assistant = assistant(
+        system_prompt="You are a specialized sub-assistant for calculations",
+    )
+
+    # Create parent assistant with sub-assistant
+    parent_assistant = assistant(
+        system_prompt="You are a parent assistant that delegates to sub-assistants",
+        sub_assistants_ids=[sub_assistant.id],
+    )
+
+    # Verify parent has sub-assistant
+    assert_that(
+        assistant_utils.get_assistant_by_id(parent_assistant.id).assistant_ids,
+        has_items(sub_assistant.id),
+        "Parent assistant should contain sub-assistant ID",
+    )
+
+
+@pytest.mark.assistant
+@pytest.mark.sub_assistant
+@pytest.mark.api
+@pytest.mark.smoke
+def test_create_assistant_with_multiple_sub_assistants(assistant, assistant_utils):
+    """
+    Test creating an assistant with multiple sub-assistants.
+
+    Verifies that:
+    - Parent assistant can reference multiple sub-assistants
+    - All sub-assistant IDs are properly stored
+    - Multiple sub-assistants can coexist
+    """
+    # Create multiple sub-assistants
+    sub_assistant_1 = assistant(
+        system_prompt="You are a specialized sub-assistant for math",
+    )
+
+    sub_assistant_2 = assistant(
+        system_prompt="You are a specialized sub-assistant for text analysis",
+    )
+
+    sub_assistant_3 = assistant(
+        system_prompt="You are a specialized sub-assistant for data processing",
+    )
+
+    # Create parent assistant with multiple sub-assistants
+    parent_assistant = assistant(
+        system_prompt="You are a parent assistant coordinating multiple sub-assistants",
+        sub_assistants_ids=[sub_assistant_1.id, sub_assistant_2.id, sub_assistant_3.id],
+    )
+
+    # Verify parent has all sub-assistants
+    parent_assistant = assistant_utils.get_assistant_by_id(parent_assistant.id)
+    assert_that(
+        len(parent_assistant.assistant_ids),
+        equal_to(3),
+        "Parent assistant should have 3 sub-assistants",
+    )
+    assert_that(
+        parent_assistant.assistant_ids,
+        has_items(sub_assistant_1.id, sub_assistant_2.id, sub_assistant_3.id),
+        "Parent assistant should contain all sub-assistant IDs",
+    )
+
+
+@pytest.mark.assistant
+@pytest.mark.sub_assistant
+@pytest.mark.api
+def test_update_assistant_to_add_sub_assistant(assistant, assistant_utils):
+    """
+    Test updating an existing assistant to add a sub-assistant.
+
+    Verifies that:
+    - Existing assistant can be updated to include sub-assistants
+    - Sub-assistant list can be modified after creation
+    """
+    # Create assistants
+    sub_assistant = assistant(
+        system_prompt="You are a specialized sub-assistant",
+    )
+
+    parent_assistant = assistant(
+        system_prompt="You are a parent assistant",
+    )
+
+    # Verify parent initially has no sub-assistants
+    parent_assistant = assistant_utils.get_assistant_by_id(parent_assistant.id)
+    assert_that(
+        len(parent_assistant.assistant_ids),
+        equal_to(0),
+        "Parent should initially have no sub-assistants",
+    )
+
+    # Update parent to include sub-assistant
+    update_request = AssistantUpdateRequest(
+        name=parent_assistant.name,
+        description=parent_assistant.description,
+        shared=False,
+        system_prompt=parent_assistant.system_prompt,
+        project=PROJECT,
+        llm_model_type=parent_assistant.llm_model_type,
+        assistant_ids=[sub_assistant.id],
+    )
+
+    assistant_utils.update_assistant(parent_assistant.id, update_request)
+
+    # Verify sub-assistant was added
+    updated_parent = assistant_utils.get_assistant_by_id(parent_assistant.id)
+    assert_that(
+        updated_parent["assistant_ids"],
+        has_items(sub_assistant.id),
+        "Parent assistant should now contain sub-assistant ID",
+    )
+
+
+@pytest.mark.assistant
+@pytest.mark.sub_assistant
+@pytest.mark.api
+def test_update_assistant_to_remove_sub_assistant(assistant, assistant_utils):
+    """
+    Test updating an assistant to remove a sub-assistant.
+
+    Verifies that:
+    - Sub-assistants can be removed from parent assistant
+    - Sub-assistant list can be cleared via update
+    """
+    # Create sub-assistant
+    sub_assistant = assistant(
+        system_prompt="You are a specialized sub-assistant",
+    )
+
+    # Create parent with sub-assistant
+    parent_assistant = assistant(
+        system_prompt="You are a parent assistant",
+        sub_assistants_ids=[sub_assistant.id],
+    )
+
+    # Verify sub-assistant was added
+    parent_assistant = assistant_utils.get_assistant_by_id(parent_assistant.id)
+    assert_that(
+        len(parent_assistant.assistant_ids),
+        equal_to(1),
+        "Parent should have one sub-assistant",
+    )
+
+    # Remove sub-assistant
+    update_request = AssistantUpdateRequest(
+        name=parent_assistant.name,
+        slug=parent_assistant.slug,
+        description=parent_assistant.description,
+        shared=False,
+        system_prompt=parent_assistant.system_prompt,
+        project=PROJECT,
+        llm_model_type=parent_assistant.llm_model_type,
+        assistant_ids=[],
+    )
+    assistant_utils.update_assistant(parent_assistant.id, update_request)
+
+    # Verify sub-assistant was removed
+    updated_parent = assistant_utils.get_assistant_by_id(parent_assistant.id)
+    assert_that(
+        len(updated_parent["assistant_ids"]),
+        equal_to(0),
+        "Parent should have no sub-assistants after removal",
+    )
+
+
+@pytest.mark.assistant
+@pytest.mark.sub_assistant
+@pytest.mark.api
+def test_nested_sub_assistants_not_allowed(assistant, assistant_utils):
+    """
+    Test that nested sub-assistant hierarchies are not allowed.
+
+    Verifies that:
+    - System prevents or rejects nested sub-assistant structures
+    - An assistant that is already a sub-assistant cannot be added as a parent
+    - Only flat (single-level) sub-assistant relationships are supported
+    """
+
+    assistant_c = assistant(
+        system_prompt="You are assistant C",
+    )
+
+    # First, make C a sub-assistant of B (B -> C)
+    assistant_b = assistant(
+        system_prompt="You are assistant B", sub_assistants_ids=[assistant_c.id]
+    )
+
+    # Verify B has C as sub-assistant
+    assistant_b = assistant_utils.get_assistant_by_id(assistant_b.id)
+    assert_that(
+        assistant_b["assistant_ids"],
+        has_items(assistant_c.id),
+        "Assistant B should have C as sub-assistant",
+    )
+
+    # Now try to create assistant with sub-assistant B (which has sub-assistant)
+    with pytest.raises(Exception) as exec_info:
+        assistant_utils.send_create_assistant_request(
+            system_prompt="You are assistant A",
+            assistant_ids=[assistant_b.id],
+        )
+    assert_error_details(
+        exec_info.value.response,
+        400,
+        f"Nested assistants not supported. Inner assistants ({assistant_b.name}) cannot contain their own inner assistants",
+    )
+
+    assistant_a = assistant(
+        system_prompt="You are assistant A",
+    )
+
+    assistant_a = assistant_utils.get_assistant_by_id(assistant_a.id)
+
+    update_a = AssistantUpdateRequest(
+        name=assistant_a.name,
+        description=assistant_a.description,
+        shared=False,
+        system_prompt=assistant_a.system_prompt,
+        project=PROJECT,
+        llm_model_type=assistant_a.llm_model_type,
+        assistant_ids=[assistant_b.id],  # Try to add B (which has C) as sub-assistant
+    )
+
+    with pytest.raises(Exception) as exec_info:
+        assistant_utils.update_assistant(assistant_a.id, update_a)
+    assert_error_details(
+        exec_info.value.response,
+        400,
+        f"Nested assistants not supported. Inner assistants ({assistant_b.name}) cannot contain their own inner assistants",
+    )
+
+
+@pytest.mark.assistant
+@pytest.mark.sub_assistant
+@pytest.mark.api
+def test_chat_with_assistant_having_different_sub_assistants(
+    assistant,
+    assistant_utils,
+    similarity_check,
+    jira_integration,
+    gitlab_integration,
+    code_datasource,
+    code_context,
+):
+    """
+    Test chatting with an assistant that has sub-assistants with different tools.
+
+    Verifies that:
+    - Parent assistant can delegate to sub-assistants with different capabilities
+    - Sub-assistants with Jira, Git, and Code-executor tools work correctly
+    - Chat functionality properly routes requests to appropriate sub-assistants
+    """
+
+    # 1. Create sub-assistant with Jira tool
+    jira_assistant = assistant(
+        Toolkit.PROJECT_MANAGEMENT,
+        ProjectManagementTool.JIRA,
+        description="Jira expert. Helps with searching and managing Jira issues.",
+        settings=jira_integration,
+        system_prompt="You are a Jira specialist assistant. You help with searching and managing Jira issues.",
+    )
+
+    # 2. Create sub-assistant with Git tool
+    git_assistant = assistant(
+        Toolkit.GIT,
+        GitTool.LIST_BRANCHES_IN_REPO,
+        description="Git expert. Helps with Git repository operations.",
+        settings=gitlab_integration,
+        context=code_context(code_datasource),
+        system_prompt="You are a Git specialist assistant. You help with Git repository operations.",
+    )
+
+    # 3. Create sub-assistant with Code-executor tool (Python REPL)
+    code_executor_assistant = assistant(
+        Toolkit.FILE_MANAGEMENT,
+        FileManagementTool.PYTHON_CODE_INTERPRETER,
+        description="Python code execution expert. Helps with running Python code.",
+        system_prompt="You are a Python code execution specialist. You help with running Python code.",
+    )
+
+    # 4. Create parent assistant with all sub-assistants
+    parent_assistant = assistant(
+        system_prompt=(
+            "You are a coordinator assistant that delegates tasks to specialized sub-assistants. "
+            "You have access to sub-assistants for Jira operations, Git operations, and Python code execution. "
+            f"Use {jira_assistant.name} for Jira related requests."
+            f"Use {git_assistant.name} for Git related requests."
+            f"Use {code_executor_assistant.name} for Code execution requests."
+        ),
+        sub_assistants_ids=[
+            jira_assistant.id,
+            git_assistant.id,
+            code_executor_assistant.id,
+        ],
+    )
+
+    # 5. Send a chat request to parent assistant asking Jira-related question
+    response = assistant_utils.ask_assistant(
+        parent_assistant, JIRA_TOOL_PROMPT, minimal_response=True
+    )
+    similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
+
+    # 6. Send a chat request to parent assistant asking Git-related question
+    git_tool_prompt = list_branches_set_active_branch_test_data[0][2]
+    git_tool_answer = list_branches_set_active_branch_test_data[0][3]
+    response = assistant_utils.ask_assistant(
+        parent_assistant, git_tool_prompt, minimal_response=True
+    )
+
+    similarity_check.check_similarity(response, git_tool_answer)
+
+    # 7. Send a chat request to parent assistant asking Code-executor question
+    response = assistant_utils.ask_assistant(
+        parent_assistant, CODE_INTERPRETER_TOOL_TASK, minimal_response=True
+    )
+
+    similarity_check.check_similarity(response, RESPONSE_FOR_CODE_INTERPRETER)

codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py

@@ -1,12 +1,8 @@
 import pytest
+from codemie_sdk.models.integration import CredentialTypes
 from hamcrest import assert_that, equal_to
 
-from codemie_sdk.models.integration import CredentialTypes
 from codemie_test_harness.tests.enums.tools import Toolkit, NotificationTool
-from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.enums.environment import Environment
-from codemie_test_harness.tests.utils.env_resolver import get_environment
-from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_TOOL_PROMPT,
     EMAIL_RESPONSE,
@@ -15,6 +11,9 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     TELEGRAM_TOOL_PROMPT,
     TELEGRAM_RESPONSE,
 )
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
+from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
 @pytest.mark.assistant
@@ -22,7 +21,7 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
 @pytest.mark.email
 @pytest.mark.api
 @pytest.mark.skipif(
-    get_environment() in [Environment.LOCALHOST, Environment.GCP],
+    EnvironmentResolver.is_localhost(),
     reason="Skipping this test on local environment",
 )
 def test_assistant_with_email_tool(

codemie_test_harness/tests/conftest.py

@@ -595,6 +595,7 @@ def assistant(default_llm, assistant_utils, conversation_utils):
         project_name: str = None,
         description: str = None,
         system_prompt="You are a helpful integration test assistant",
+        sub_assistants_ids: List[str] = None,
     ):
         nonlocal created_assistant
         # Correctly handle empty `tool_names`
@@ -622,6 +623,7 @@ def assistant(default_llm, assistant_utils, conversation_utils):
             system_prompt=system_prompt,
             project_name=project_name,
             description=description,
+            assistant_ids=sub_assistants_ids if sub_assistants_ids else [],
         )
         return created_assistant
 

codemie_test_harness/tests/llm/assistants/test_llm.py

@@ -1,6 +1,9 @@
+from typing import Sequence
+
 import pytest
 from codemie_sdk.models.assistant import ToolKitDetails, ToolDetails
 from hamcrest import assert_that, has_item
+
 from codemie_test_harness.tests.enums.model_types import ModelTypes
 from codemie_test_harness.tests.enums.tools import Toolkit, FileManagementTool
 from codemie_test_harness.tests.test_data.llm_test_data import MODEL_RESPONSES
@@ -8,6 +11,13 @@ from codemie_test_harness.tests.utils.client_factory import get_client
 from codemie_test_harness.tests.utils.env_resolver import get_environment
 from codemie_test_harness.tests.utils.pytest_utils import check_mark
 
+SIMPLE_GREETING_PROMPT = "Just say one word: 'Hello'"
+
+
+def get_model_names(llm_utils) -> Sequence[str]:
+    """Get list of available model names."""
+    return [row.base_name for row in llm_utils.list_llm_models()]
+
 
 def pytest_generate_tests(metafunc):
     if "model_type" in metafunc.fixturenames:
@@ -41,7 +51,7 @@ def test_assistant_with_different_models(
     llm_utils, assistant_utils, model_type, similarity_check, filesystem_integration
 ):
     assert_that(
-        [row.base_name for row in llm_utils.list_llm_models()],
+        get_model_names(llm_utils),
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
@@ -56,8 +66,8 @@ def test_assistant_with_different_models(
         settings=filesystem_integration,
     )
 
-    assistant = assistant_utils.create_assistant(model_type, toolkits=[tool])
-    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+    _assistant = assistant_utils.create_assistant(model_type, toolkits=[tool])
+    response = assistant_utils.ask_assistant(_assistant, SIMPLE_GREETING_PROMPT)
 
     if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
         response = "\n".join(response.split("\n")[-3:])
@@ -73,12 +83,12 @@ def test_assistant_with_different_models_with_top_p_parameter(
     llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in llm_utils.list_llm_models()],
+        get_model_names(llm_utils),
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
-    assistant = assistant_utils.create_assistant(model_type, top_p=0.5)
-    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+    _assistant = assistant_utils.create_assistant(model_type, top_p=0.5)
+    response = assistant_utils.ask_assistant(_assistant, SIMPLE_GREETING_PROMPT)
 
     if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
         response = "\n".join(response.split("\n")[-3:])
@@ -94,12 +104,12 @@ def test_assistant_with_different_models_with_temperature_parameter(
     llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in llm_utils.list_llm_models()],
+        get_model_names(llm_utils),
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
-    assistant = assistant_utils.create_assistant(model_type, temperature=0.5)
-    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+    _assistant = assistant_utils.create_assistant(model_type, temperature=0.5)
+    response = assistant_utils.ask_assistant(_assistant, SIMPLE_GREETING_PROMPT)
 
     if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
         response = "\n".join(response.split("\n")[-3:])
@@ -121,15 +131,49 @@ def test_assistant_with_different_models_with_datasource_attached(
     file_datasource,
 ):
     assert_that(
-        [row.base_name for row in llm_utils.list_llm_models()],
+        get_model_names(llm_utils),
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
 
-    assistant = assistant_utils.create_assistant(
+    _assistant = assistant_utils.create_assistant(
         model_type, context=[kb_context(file_datasource)]
     )
-    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+    response = assistant_utils.ask_assistant(_assistant, SIMPLE_GREETING_PROMPT)
+
+    if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
+        response = "\n".join(response.split("\n")[-3:])
+    similarity_check.check_similarity(response, "Hello")
+
+
+@pytest.mark.assistant
+@pytest.mark.llm
+@pytest.mark.sub_assistant
+@pytest.mark.api
+@pytest.mark.smoke
+def test_assistant_with_different_models_with_sub_assistant(
+    llm_utils, assistant_utils, model_type, similarity_check
+):
+    assert_that(
+        get_model_names(llm_utils),
+        has_item(model_type),
+        f"{model_type} is missing in backend response",
+    )
+
+    # Create sub-assistant with default model
+    sub_assistant = assistant_utils.create_assistant(
+        system_prompt="You are a specialized sub-assistant.",
+    )
+
+    # Create parent assistant with the parametrized model that delegates to sub-assistant
+    parent_assistant = assistant_utils.create_assistant(
+        llm_model_type=model_type,
+        system_prompt="You are a coordinator assistant.",
+        assistant_ids=[sub_assistant.id],
+    )
+
+    # Test parent assistant that should delegate to sub-assistant
+    response = assistant_utils.ask_assistant(parent_assistant, SIMPLE_GREETING_PROMPT)
 
     if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
         response = "\n".join(response.split("\n")[-3:])

codemie_test_harness/tests/test_data/direct_tools/notification_tools_test_data.py

@@ -1,6 +1,3 @@
-from codemie_test_harness.tests.enums.environment import Environment
-from codemie_test_harness.tests.utils.env_resolver import get_environment
-
 import pytest
 
 from codemie_test_harness.tests.enums.tools import NotificationTool
@@ -9,6 +6,7 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_BODY,
     EMAIL_RESPONSE,
 )
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 notification_tools_test_data = [
     pytest.param(
@@ -22,7 +20,7 @@ notification_tools_test_data = [
         marks=[
             pytest.mark.email,
             pytest.mark.skipif(
-                get_environment() in [Environment.LOCALHOST, Environment.GCP],
+                EnvironmentResolver.is_localhost(),
                 reason="Skipping this test on local environment",
             ),
         ],

codemie_test_harness/tests/test_data/pm_tools_test_data.py

@@ -2,7 +2,7 @@ from codemie_test_harness.tests.utils.confluence_utils import CONFLUENCE_SPACE_K
 from codemie_test_harness.tests.utils.jira_utils import JIRA_PROJECT_KEYS
 
 JIRA_TOOL_PROMPT = (
-    "Get a title for ticket EPMCDME-222 ticket."
+    "Get a title for EPMCDME-222 ticket."
     "For generic jira tool use exactly the same parameters:"
     "relative_url=/rest/api/2/issue/EPMCDME-222"
     "method=GET"

codemie_test_harness/tests/utils/assistant_utils.py

@@ -30,6 +30,7 @@ class AssistantUtils(BaseUtils):
         project_name=None,
         top_p=None,
         temperature=None,
+        assistant_ids=(),
     ):
         # Generate a random name if assistant_name is not provided
         assistant_name = assistant_name if assistant_name else get_random_name()
@@ -50,6 +51,7 @@ class AssistantUtils(BaseUtils):
             mcp_servers=mcp_servers,
             top_p=top_p,
             temperature=temperature,
+            assistant_ids=list(assistant_ids) if assistant_ids else [],
         )
 
         response = self.client.assistants.create(request)
@@ -70,6 +72,7 @@ class AssistantUtils(BaseUtils):
         top_p=None,
         temperature=None,
         description=None,
+        assistant_ids=(),
     ):
         # Generate a random name if assistant_name is not provided
         assistant_name = assistant_name if assistant_name else get_random_name()
@@ -91,6 +94,7 @@ class AssistantUtils(BaseUtils):
             top_p=top_p,
             temperature=temperature,
             description=description,
+            assistant_ids=assistant_ids,
         )
 
         return wait_for_entity(

codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py

@@ -1,6 +1,3 @@
-from codemie_test_harness.tests.enums.environment import Environment
-from codemie_test_harness.tests.utils.env_resolver import get_environment
-
 import pytest
 from hamcrest import assert_that, equal_to
 
@@ -14,6 +11,7 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     TELEGRAM_RESPONSE,
 )
 from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
 @pytest.mark.workflow
@@ -22,7 +20,7 @@ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 @pytest.mark.email
 @pytest.mark.api
 @pytest.mark.skipif(
-    get_environment() in [Environment.LOCALHOST, Environment.GCP],
+    EnvironmentResolver.is_localhost(),
     reason="Skipping this test on local environment",
 )
 def test_workflow_with_email_tool(

codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_notification_tools.py

@@ -39,7 +39,7 @@ def test_workflow_with_notification_tools_direct(
         _workflow.id, tool_and_state_name, user_input=json.dumps(prompt)
     )
 
-    similarity_check.check_similarity(response, expected_response, 80)
+    similarity_check.check_similarity(response, expected_response, 75)
 
 
 @pytest.mark.api
@@ -65,7 +65,7 @@ def test_workflow_with_notification_tools_with_hardcoded_args(
     )
     response = workflow_utils.execute_workflow(_workflow.id, tool_and_state_name)
 
-    similarity_check.check_similarity(response, expected_response, 80)
+    similarity_check.check_similarity(response, expected_response, 75)
 
 
 @pytest.mark.api

codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py

@@ -2,7 +2,6 @@ import pytest
 from hamcrest import assert_that, equal_to
 
 from codemie_test_harness.tests.enums.tools import NotificationTool
-from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_TOOL_PROMPT,
     EMAIL_RESPONSE,
@@ -11,9 +10,9 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     TELEGRAM_TOOL_PROMPT,
     TELEGRAM_RESPONSE,
 )
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.utils.base_utils import get_random_name
-from codemie_test_harness.tests.enums.environment import Environment
-from codemie_test_harness.tests.utils.env_resolver import get_environment
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
 @pytest.mark.workflow
@@ -23,7 +22,7 @@ from codemie_test_harness.tests.utils.env_resolver import get_environment
 @pytest.mark.api
 @pytest.mark.testcase("EPMCDME-6652")
 @pytest.mark.skipif(
-    get_environment() in [Environment.LOCALHOST, Environment.GCP],
+    EnvironmentResolver.is_localhost(),
     reason="Skipping this test on local environment",
 )
 def test_workflow_with_notification_email_tool(

{codemie_test_harness-0.1.218.dist-info → codemie_test_harness-0.1.221.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.218
+Version: 0.1.221
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.218)
+Requires-Dist: codemie-sdk-python (==0.1.221)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)

{codemie_test_harness-0.1.218.dist-info → codemie_test_harness-0.1.221.dist-info}/RECORD

@@ -26,6 +26,7 @@ codemie_test_harness/tests/assistant/default_integrations/test_default_integrati
 codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_kit.py,sha256=k455DvSPG-mIyk2vuJfp-hJVUsUUyWlOBlb-igMJi2E,8511
 codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_with_datasource.py,sha256=knJ86qJLvMozHPLXMeFzeXrJJVIleqNQte9zSw6b0oI,10831
 codemie_test_harness/tests/assistant/test_assistants.py,sha256=pFcTM8BjuOYnOKu_qwai1NVdsG0ztWYYNQuOkivPS7A,16609
+codemie_test_harness/tests/assistant/test_sub_assistants.py,sha256=BY-kbq2I1I3vSvf0pDBU8LLKwGqyQnuhc4KqlMx0nvA,12657
 codemie_test_harness/tests/assistant/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/assistant/tools/access_management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py,sha256=FVCb9VBUtwApiBkrqDCSpelZvPRXRR4LmbrNr1wk13s,941
@@ -47,7 +48,7 @@ codemie_test_harness/tests/assistant/tools/mcp/__init__.py,sha256=47DEQpj8HBSa-_
 codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py,sha256=SCEf5koVICJYRv5Du3TT0viAPmWojgHO8PX5ZVGPCrE,2761
 codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py,sha256=omH5BVtyOZqlqX9NOfWelJ93geLSTVQXropCMAhAkiA,1568
 codemie_test_harness/tests/assistant/tools/notification/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py,sha256=4XrHYvUtbm8bp2bmajOuSSgvSuwXI8etHK6vRhIWkDM,2962
+codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py,sha256=ils26CN9ZUh6s5D48uuz3IGEDtilR7uRl64cQFCa61U,2870
 codemie_test_harness/tests/assistant/tools/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py,sha256=DXyCNON0EmrTqDbg-su9pPe46dS4EB5UasuHBNwMn-s,1262
 codemie_test_harness/tests/assistant/tools/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -64,7 +65,7 @@ codemie_test_harness/tests/assistant/tools/servicenow/__init__.py,sha256=47DEQpj
 codemie_test_harness/tests/assistant/tools/servicenow/test_servicenow_tools.py,sha256=aUjfZ4773WoQJjcHx3JqH5e8ckaKB-aIMO-OZWTm0Ek,888
 codemie_test_harness/tests/assistant/tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/assistant/tools/vcs/test_assistant_with_vcs_tools.py,sha256=qOPr4XOh2rgUV2MXMxkRzRGkAKl9ViwQGCZ-dMEtscU,1145
-codemie_test_harness/tests/conftest.py,sha256=o2A8_JrhYCIqPzBJwPtlORQexLjv8k6HpZmJNPy7An0,33807
+codemie_test_harness/tests/conftest.py,sha256=FH6xBlPTjU6wlHNZSJAxbNvYCjICZq3SRWMCEAKvF2c,33929
 codemie_test_harness/tests/conversations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/conversations/test_conversations_endpoints.py,sha256=HQ2nu9lXfRNkyJhA0rzar7Rmv6pMe-te0rFYAy-X5UA,4128
 codemie_test_harness/tests/e2e/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -84,7 +85,7 @@ codemie_test_harness/tests/integrations/user/test_user_integrations.py,sha256=SO
 codemie_test_harness/tests/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/llm/assistants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/llm/assistants/test_lite_llm.py,sha256=jYtf_J7lBSmB1LmmXWjGHkMWmpcqAJ3l_IJovgYxmZM,3683
-codemie_test_harness/tests/llm/assistants/test_llm.py,sha256=b5HhrDkz1lwCaSZH5kdPdacmLXH_Bxnj5vO_A5ho3k8,4838
+codemie_test_harness/tests/llm/assistants/test_llm.py,sha256=APpyNpdcs0ID-IYy8Yxip_y954AqEz4zQzVgpeVYa2M,6130
 codemie_test_harness/tests/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/providers/test_providers_endpoints.py,sha256=0lHGRO3mh4uVK-GUPasPc05nrL4HfaYZPUPu9TZzw04,7946
 codemie_test_harness/tests/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -123,7 +124,7 @@ codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_dat
 codemie_test_harness/tests/test_data/direct_tools/direct_tools_test_data.py,sha256=y9awl1IA6EXGXyz05QzcNdt5z7Rk9J5LzIbfi4CFE3s,3233
 codemie_test_harness/tests/test_data/direct_tools/file_management_tools_test_data.py,sha256=HD55qI37_koSBzSJBkafUI1CrgnL5hnWoA9n0cjOkmY,1622
 codemie_test_harness/tests/test_data/direct_tools/keycloak_tool_test_data.py,sha256=6lU1YC6DSV6e_3VNQEVvtA4o3_lSFgOQik4T0u-TF1g,2979
-codemie_test_harness/tests/test_data/direct_tools/notification_tools_test_data.py,sha256=Xo76Bd4R9cmfhEHUaUGh2O1qPzeoLG1El0m7VLRwP5M,1880
+codemie_test_harness/tests/test_data/direct_tools/notification_tools_test_data.py,sha256=EF8L0c7XjbueooF0Px4tdAqWje16VfXv39tNvfn3AoQ,1787
 codemie_test_harness/tests/test_data/direct_tools/open_api_tools_test_data.py,sha256=UwXVrwD3MnOIS2wwikNDk1EZNYR14p_IjhOOUwjYHOI,497
 codemie_test_harness/tests/test_data/direct_tools/project_management_tools_test_data.py,sha256=3s2biYoy5Nv8nwaWYbp1AoMpZ0lrUvlqHuFP9BlOHwg,21084
 codemie_test_harness/tests/test_data/direct_tools/report_portal_tools_test_data.py,sha256=C_3eqdhBh9bAl23fXXP0P23aN7xwMvvBVgaPC6_-ksA,53557
@@ -163,7 +164,7 @@ codemie_test_harness/tests/test_data/open_api_tools_test_data.py,sha256=mvf0legy
 codemie_test_harness/tests/test_data/openapi.json,sha256=X4uqtfjpTUuMifefQRf8mHI1k8pspp8-L0rpJlhLOI4,10459
 codemie_test_harness/tests/test_data/output_schema_test_data.py,sha256=4l7AvXbMl9hIvoFxu1LPPSGz9hb5Uz2_is4zTm77ARY,261
 codemie_test_harness/tests/test_data/plugin_tools_test_data.py,sha256=bVamztyQ4bAVo1CRSrtu6f5H-gkjhAN2nq5Jbc0erqM,4168
-codemie_test_harness/tests/test_data/pm_tools_test_data.py,sha256=yjEG24F10Rb5Br5dEPMIO74WMi2p3huZFgfRMYfzG7c,5150
+codemie_test_harness/tests/test_data/pm_tools_test_data.py,sha256=HNPxPQtHPjC0I02LmxF8dDr5orpqwtKOQ1KoHD_4_9c,5143
 codemie_test_harness/tests/test_data/project_management_test_data.py,sha256=2RWzrJmdlrOuJQKcmlWOfYz2daw_Oc2RkDU5XyM-w6U,3242
 codemie_test_harness/tests/test_data/report_portal_tools_test_data.py,sha256=YZdmfEwrwOdCduNxs768LOB8OHfL8sfNI-R2k-koKTk,14555
 codemie_test_harness/tests/test_data/research_tools_test_data.py,sha256=zwpzm-VSnrLZEfG97AE9Ms7z7j3xmqxiNd1EmZyWCSk,9102
@@ -269,7 +270,7 @@ codemie_test_harness/tests/ui/workflows/test_workflow_executions_page.py,sha256=
 codemie_test_harness/tests/ui/workflows/test_workflow_templates.py,sha256=u3EK7FkwjdGLK2JoDhc4gcVFtVXT-2-1UKBlFL-XlR0,4656
 codemie_test_harness/tests/ui/workflows/test_workflows.py,sha256=zRBFiQYhJ_MWKGGxUgGNsiTbs8P8Zw6HCuJXzkABvj0,3817
 codemie_test_harness/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-codemie_test_harness/tests/utils/assistant_utils.py,sha256=2s1MikCfIcM3UlkcuwaedyxI2hePuWT3h8F3SDjBCtk,8022
+codemie_test_harness/tests/utils/assistant_utils.py,sha256=McT6EYZVQELYeYqjkgcJR08Pk4wXEXPKoT-z8olmcVY,8187
 codemie_test_harness/tests/utils/aws_parameters_store.py,sha256=YAVpvwElkKZJZvzSVxtOue1Gjs-kvSBS2y5QvIlz484,3267
 codemie_test_harness/tests/utils/base_utils.py,sha256=Yyj9HUk8-3Wf0hpWrMXiWDjvuw7ZRt89wiYUkoDghLk,7203
 codemie_test_harness/tests/utils/client_factory.py,sha256=xGta0ZaVYzWfwJ4cu3f89KkGc_R5Bq-9lqnhr57x_2w,972
@@ -324,7 +325,7 @@ codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assis
 codemie_test_harness/tests/workflow/assistant_tools/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py,sha256=_tj5IP5Qy0tNI69hdDd5Cuecwu4Xz5dyrMCdvfnRQDs,3330
 codemie_test_harness/tests/workflow/assistant_tools/notification/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py,sha256=0LztefJikX9looSkNzHgCW_ocj6B6EWQsSlooTBntl8,3120
+codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py,sha256=zwbWIOVxKnkoTGJTq39gpJmUKqJMQOwzbqgJds72QMA,3027
 codemie_test_harness/tests/workflow/assistant_tools/open_api/__init__.py,sha256=s6X-VOfd5UR7yxRL90VvL21YdYs1rPd6wkGFSfOI3Qs,46
 codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py,sha256=CtvQx6K1Nr821KusEfM1bqCjpm6IglCHQD7iaHgCXQw,1448
 codemie_test_harness/tests/workflow/assistant_tools/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -355,7 +356,7 @@ codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_code
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools_elastic.py,sha256=NzPxNdxLYAySIDzsP1ttF4BiDuk9kibHipFpqavKHAA,3271
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools_sql.py,sha256=8Clmo81Kd6Y_sEn532f4nhsIYvGy8REDGCPiFZyvPno,4161
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_file_management_tools.py,sha256=Ii7b65srfwk16jYALy5BzMhVErE3GPBWZ2M3JLkjqo0,2880
-codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_notification_tools.py,sha256=YzhsmmfidBYsGymBoO3q2WM9zw2zZfq7uOj90-bPN0c,2983
+codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_notification_tools.py,sha256=AUHfqOFHDmYZlMeSx1sQ-V1ia2caQ8JExqb-xHLka3M,2983
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_open_api_tools.py,sha256=CkF5VikbCMQyGoeibF9Al50AN-GW83F6k1usow17YlY,3208
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_plugin_tools.py,sha256=k8k0WlR0nJwr7OP61vHZ3a2Zvn9g4Gy2W1boiuiBXG0,4106
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_project_management_tools.py,sha256=BiMmDw5f7KNoYH1IfNX-7syV6CJV-4kP1U4OTSRJ6u4,3405
@@ -388,7 +389,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_wi
 codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py,sha256=qIIrKKIa0copWSUOcP87_YEFV1QpKs7OqsAAcVXF2uM,3560
 codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/__init__.py,sha256=hUDnGT_v3FoV6qsGpdACg_DfBFlfuubj8FjZLiuMrt0,50
-codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py,sha256=UvULHw9k-GhrwBHwj3_s_2UVwh0nZgzu0YG0ZzlZMow,3193
+codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py,sha256=SMHB1EPJ6aqsZsJgLzzk7OjT7UL_X20Or5l5bpYnsJE,3101
 codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/__init__.py,sha256=s6X-VOfd5UR7yxRL90VvL21YdYs1rPd6wkGFSfOI3Qs,46
 codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py,sha256=JHZAtbWXLsgI6R0IYzw1yrZ8Pn5KtdD2KgbpokVcJjE,1474
 codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -404,7 +405,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/__init__.
 codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py,sha256=D835gaRbCnB4va5mi9TdA_u9StSpGXQ_fgzwW0S2pwo,1173
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py,sha256=Se9imIiBYuJU78m1pLu0g4ZmHygKZjr6JjIWkGXTy1Q,1364
-codemie_test_harness-0.1.218.dist-info/METADATA,sha256=EmA3TMXO9FmRr4mF2hWy9Mcsu4AoEiA0tgjKDJUsdPM,27184
-codemie_test_harness-0.1.218.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-codemie_test_harness-0.1.218.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
-codemie_test_harness-0.1.218.dist-info/RECORD,,
+codemie_test_harness-0.1.221.dist-info/METADATA,sha256=OuLbVODiKmBvq1nHcaT-popN7ecY22G6Mcci97oQCOc,27184
+codemie_test_harness-0.1.221.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+codemie_test_harness-0.1.221.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
+codemie_test_harness-0.1.221.dist-info/RECORD,,