codemie-test-harness 0.1.158__py3-none-any.whl → 0.1.160__py3-none-any.whl

codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py

@@ -38,15 +38,16 @@ def confluence_cloud_datasource(datasource_utils, confluence_cloud_integration):
             CONFLUENCE_TOOL_PROMPT,
             RESPONSE_FOR_CONFLUENCE_TOOL,
             marks=pytest.mark.confluence,
+            id="Confluence",
         ),
         pytest.param(
             "confluence_cloud_datasource",
             CONFLUENCE_CLOUD_TOOL_PROMPT,
             RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL,
             marks=[pytest.mark.confluence, pytest.mark.confluence_cloud],
+            id="Confluence Cloud",
         ),
     ],
-    ids=["Confluence", "Confluence Cloud"],
 )
 def test_create_datasource_with_confluence_and_confluence_cloud_integration(
     request,

codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py

@@ -36,15 +36,16 @@ def jira_cloud_datasource(datasource_utils, jira_cloud_integration):
             JIRA_TOOL_PROMPT,
             RESPONSE_FOR_JIRA_TOOL,
             marks=pytest.mark.jira,
+            id="Jira",
         ),
         pytest.param(
             "jira_cloud_datasource",
             JIRA_CLOUD_TOOL_PROMPT,
             RESPONSE_FOR_JIRA_CLOUD_TOOL,
             marks=[pytest.mark.jira, pytest.mark.jira_cloud],
+            id="Jira Cloud",
         ),
     ],
-    ids=["Jira", "Jira Cloud"],
 )
 def test_create_datasource_with_jira_and_jira_cloud_integration(
     request,

codemie_test_harness/tests/assistant/tools/cloud/test_cloud_tools.py

@@ -1,6 +1,5 @@
 import pytest
 
-from codemie_test_harness.tests.enums.tools import Toolkit, CloudTool
 from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
 
 
@@ -10,12 +9,6 @@ from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_tes
 @pytest.mark.parametrize(
     "toolkit,tool_name,credential_type,credentials,prompt,expected_response",
     cloud_test_data,
-    ids=[
-        f"{Toolkit.CLOUD}_{CloudTool.AWS}",
-        f"{Toolkit.CLOUD}_{CloudTool.AZURE}",
-        f"{Toolkit.CLOUD}_{CloudTool.GCP}",
-        f"{Toolkit.CLOUD}_{CloudTool.KUBERNETES}",
-    ],
 )
 def test_assistant_with_cloud_tools(
     assistant_utils,

codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py

@@ -39,7 +39,6 @@ def test_assistant_with_codebase_tools(
 @pytest.mark.parametrize(
     "toolkit, tool_name, credentials, prompt, expected",
     sonar_tools_test_data,
-    ids=[f"{row[0]}_{row[1]}" for row in sonar_tools_test_data],
 )
 def test_assistant_with_sonar_tools(
     assistant_utils,

codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py

@@ -1,10 +1,8 @@
-import os
 import uuid
 
 import pytest
 from codemie_sdk.models.integration import CredentialTypes
 
-from codemie_test_harness.tests.enums.integrations import DataBaseDialect
 from codemie_test_harness.tests.enums.tools import (
     Toolkit,
     DataManagementTool,
@@ -20,9 +18,11 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
     RESPONSE_FOR_SQL,
 )
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 pytestmark = pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this tests on local environment"
+    EnvironmentResolver.is_localhost(),
+    reason="Skipping this tests on local environment",
 )
 
 
@@ -55,11 +55,10 @@ def test_create_assistant_with_elastic_tool(
 @pytest.mark.parametrize(
     "db_dialect",
     sql_tools_test_data,
-    ids=[DataBaseDialect.MY_SQL, DataBaseDialect.POSTGRES, DataBaseDialect.MS_SQL],
 )
 @pytest.mark.testcase("EPMCDME-6132")
 @pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
+    EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
 )
 def test_create_assistant_with_sql_tool(
     integration_utils, assistant_utils, assistant, similarity_check, db_dialect

codemie_test_harness/tests/assistant/tools/filemanagement/test_assistant_with_file_management_tools.py

@@ -1,5 +1,3 @@
-import os
-
 import pytest
 from hamcrest import assert_that, contains_string, is_not, all_of
 
@@ -17,6 +15,7 @@ from codemie_test_harness.tests.test_data.file_management_tools_test_data import
     show_diff_task,
 )
 from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
 @pytest.mark.assistant
@@ -26,12 +25,6 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
 @pytest.mark.parametrize(
     "tool_name,prompt,expected_response",
     file_management_tools_test_data,
-    ids=[
-        FileManagementTool.PYTHON_CODE_INTERPRETER,
-        FileManagementTool.LIST_DIRECTORY,
-        FileManagementTool.WRITE_FILE,
-        FileManagementTool.RUN_COMMAND_LINE,
-    ],
 )
 def test_create_assistant_with_file_management_tool(
     filesystem_integration,
@@ -56,7 +49,7 @@ def test_create_assistant_with_file_management_tool(
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-6103")
 @pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
+    EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
 )
 def test_create_assistant_with_file_management_generate_image_tool(
     filesystem_integration, assistant, assistant_utils, similarity_check

codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py

@@ -1,4 +1,3 @@
-import os
 from copy import deepcopy
 
 import pytest
@@ -8,19 +7,21 @@ from hamcrest import assert_that, contains_string
 from codemie_sdk.models.integration import CredentialTypes
 from codemie_test_harness.tests import autotest_entity_prefix
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     cli_mcp_server_test_data,
     CLI_MCP_SERVER,
 )
 
+pytestmark = pytest.mark.skipif(
+    EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
+)
+
 
 @pytest.mark.assistant
 @pytest.mark.mcp
 @pytest.mark.regression
-@pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
-)
 @pytest.mark.parametrize(
     "command,expected_answer",
     cli_mcp_server_test_data,
@@ -41,9 +42,6 @@ def test_cli_mcp_server(
 @pytest.mark.assistant
 @pytest.mark.mcp
 @pytest.mark.regression
-@pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
-)
 def test_env_var_in_mcp_server(
     assistant_utils, assistant, similarity_check, integration_utils
 ):

codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py

@@ -1,4 +1,4 @@
-import os
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 import pytest
 
@@ -11,13 +11,14 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     fetch_server_prompt,
 )
 
+pytestmark = pytest.mark.skipif(
+    EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
+)
+
 
 @pytest.mark.assistant
 @pytest.mark.mcp
 @pytest.mark.regression
-@pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
-)
 @pytest.mark.parametrize(
     "mcp_server",
     time_mcp_server_test_data,
@@ -41,9 +42,6 @@ def test_creation_mcp_server_with_form_configuration(
 @pytest.mark.assistant
 @pytest.mark.mcp
 @pytest.mark.regression
-@pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
-)
 def test_fetch_mcp_server(assistant_utils, assistant, similarity_check):
     assistant = assistant(mcp_server=FETCH_MCP_SERVER)
 

codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py

@@ -1,11 +1,11 @@
-import os
-
 import pytest
 from hamcrest import assert_that, equal_to
 
 from codemie_sdk.models.integration import CredentialTypes
 from codemie_test_harness.tests.enums.tools import Toolkit, NotificationTool
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.enums.environment import Environment
+from codemie_test_harness.tests.utils.env_resolver import get_environment
 from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_TOOL_PROMPT,
     EMAIL_RESPONSE,
@@ -21,7 +21,7 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
 @pytest.mark.email
 @pytest.mark.regression
 @pytest.mark.skipif(
-    os.getenv("ENV") in ["local", "gcp"],
+    get_environment() in [Environment.LOCALHOST, Environment.GCP],
     reason="Skipping this test on local environment",
 )
 def test_assistant_with_email_tool(

codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py

@@ -1,4 +1,4 @@
-import os
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 import pytest
 
@@ -13,7 +13,8 @@ from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-6127")
 @pytest.mark.skipif(
-    os.getenv("ENV") == "azure", reason="Still have an issue with encoding long strings"
+    EnvironmentResolver.is_azure(),
+    reason="Still have an issue with encoding long strings",
 )
 @pytest.mark.parametrize(
     "tool_name,prompt,expected_response",

codemie_test_harness/tests/assistant/tools/report_portal/test_assistant_report_portal_tools.py

@@ -0,0 +1,32 @@
+import pytest
+
+from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
+    rp_test_data,
+)
+
+
+@pytest.mark.assistant
+@pytest.mark.report_portal
+@pytest.mark.regression
+@pytest.mark.parametrize(
+    "toolkit,tool_name,prompt,expected_response",
+    rp_test_data,
+    ids=[f"{row[0]}_{row[1]}" for row in rp_test_data],
+)
+def test_assistant_with_report_portal_tools(
+    assistant,
+    assistant_utils,
+    report_portal_integration,
+    similarity_check,
+    toolkit,
+    tool_name,
+    prompt,
+    expected_response,
+):
+    assistant = assistant(
+        toolkit,
+        tool_name,
+        settings=report_portal_integration,
+    )
+    response = assistant_utils.ask_assistant(assistant, prompt)
+    similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/conftest.py

@@ -52,6 +52,10 @@ from codemie_test_harness.tests.utils.yaml_utils import (
     StateModel,
     prepare_yaml_content,
 )
+from codemie_test_harness.tests.utils.env_resolver import (
+    EnvironmentResolver,
+    get_environment,
+)
 
 logger = setup_logger(__name__)
 
@@ -84,7 +88,7 @@ def pytest_configure(config):
 
         # Get dotenv configuration from AWS Parameter Store
         dotenv = aws_parameters_store.get_parameter(
-            f"/codemie/autotests/dotenv/{os.getenv('ENV')}"
+            f"/codemie/autotests/dotenv/{str(get_environment())}"
         )
 
         # Safely update .env file with new content
@@ -374,6 +378,17 @@ def keycloak_integration(integration_utils):
         integration_utils.delete_integration(integration)
 
 
+@pytest.fixture(scope="module")
+def report_portal_integration(integration_utils):
+    credential_values = CredentialsUtil.report_portal_credentials()
+    integration = integration_utils.create_integration(
+        CredentialTypes.REPORT_PORTAL, credential_values
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
 @pytest.fixture(scope="function")
 def general_integration(integration_utils):
     created_integration: Optional[Integration] = None
@@ -826,7 +841,7 @@ def cloud_integration(integration_utils):
 
 def pytest_sessionfinish(session):
     """Run cleanup code after all tests have finished."""
-    clean_up_timeout = 1 if os.getenv("ENV") == "PROD" else 0
+    clean_up_timeout = 1 if EnvironmentResolver.is_production() else 0
     if os.getenv("CLEANUP_DATA", "true").lower() == "true":
         client = get_client()
         prefix = autotest_entity_prefix

codemie_test_harness/tests/enums/environment.py

@@ -0,0 +1,102 @@
+"""
+Environment enumeration for CodeMie test harness.
+
+This enum defines all supported environments for the CodeMie platform,
+providing type safety and preventing typos in environment checks.
+"""
+
+from enum import Enum
+from typing import List
+
+
+class Environment(Enum):
+    """
+    Enumeration of supported CodeMie environments.
+
+    Each environment corresponds to a specific deployment of the CodeMie platform
+    with its own domain patterns and configuration requirements.
+    """
+
+    PRODUCTION = "prod"
+    PREVIEW = "preview"
+    LOCALHOST = "localhost"
+    AZURE = "azure"
+    GCP = "gcp"
+    AWS = "aws"
+    UNKNOWN = "unknown"
+
+    def __str__(self) -> str:
+        """Return the string value of the environment."""
+        return self.value
+
+    @property
+    def is_production(self) -> bool:
+        """Check if this is the production environment."""
+        return self == Environment.PRODUCTION
+
+    @property
+    def is_preview(self) -> bool:
+        """Check if this is the preview environment."""
+        return self == Environment.PREVIEW
+
+    @property
+    def is_localhost(self) -> bool:
+        """Check if this is the localhost environment."""
+        return self == Environment.LOCALHOST
+
+    @property
+    def is_sandbox(self) -> bool:
+        """Check if this is any sandbox environment (azure, gcp, aws)."""
+        return self in (Environment.AZURE, Environment.GCP, Environment.AWS)
+
+    @property
+    def is_azure(self) -> bool:
+        """Check if this is the Azure sandbox environment."""
+        return self == Environment.AZURE
+
+    @property
+    def is_gcp(self) -> bool:
+        """Check if this is the GCP sandbox environment."""
+        return self == Environment.GCP
+
+    @property
+    def is_aws(self) -> bool:
+        """Check if this is the AWS sandbox environment."""
+        return self == Environment.AWS
+
+    @classmethod
+    def get_all_sandbox_environments(cls) -> List["Environment"]:
+        """Get all sandbox environments as enum values."""
+        return [cls.AZURE, cls.GCP, cls.AWS]
+
+    @classmethod
+    def get_all_environments(cls) -> List["Environment"]:
+        """Get all available environments."""
+        return list(cls)
+
+    @classmethod
+    def get_azure_environments(cls) -> List["Environment"]:
+        """Get environments where Azure services are available.
+
+        Returns:
+            List of Environment enums: [PREVIEW, AZURE, LOCALHOST]
+        """
+        return [cls.PREVIEW, cls.AZURE, cls.LOCALHOST]
+
+    @classmethod
+    def get_gcp_environments(cls) -> List["Environment"]:
+        """Get environments where GCP services are available.
+
+        Returns:
+            List of Environment enums: [PREVIEW, GCP, LOCALHOST]
+        """
+        return [cls.PREVIEW, cls.GCP, cls.LOCALHOST]
+
+    @classmethod
+    def get_aws_environments(cls) -> List["Environment"]:
+        """Get environments where AWS services are available.
+
+        Returns:
+            List of Environment enums: [PREVIEW, AWS, LOCALHOST]
+        """
+        return [cls.PREVIEW, cls.AWS, cls.LOCALHOST]

codemie_test_harness/tests/enums/model_types.py

@@ -34,6 +34,7 @@ class ModelTypes(str, Enum):
     CLAUDE_4_SONNET = "claude-4-sonnet"
     CLAUDE_4_OPUS = "claude-4-opus"
     CLAUDE_4_1_OPUS = "claude-4-1-opus"
+    CLAUDE_4_SONNET_1M = "claude-4-sonnet-1m"
 
     # Other
     RLAB_QWQ_32B = "rlab-qwq-32b"

codemie_test_harness/tests/enums/tools.py

@@ -20,6 +20,7 @@ class Toolkit(str, Enum):
     DATA_MANAGEMENT = "Data Management"
     SERVICENOW = "IT Service Management"
     ACCESS_MANAGEMENT = "Access Management"
+    REPORT_PORTAL = "Report Portal"
 
 
 class GitTool(str, Enum):
@@ -185,3 +186,16 @@ class AccessManagementTool(str, Enum):
     """Enum for Access Management tool names."""
 
     KEYCLOAK = "keycloak"
+
+
+class ReportPortalTool(str, Enum):
+    """Enum for Report Portal tool names."""
+
+    GET_EXTENDED_LAUNCH_DATA_AS_RAW = "get_extended_launch_data_as_raw"
+    GET_LAUNCH_DETAILS = "get_launch_details"
+    GET_ALL_LAUNCHES = "get_all_launches"
+    FIND_TEST_ITEM_BY_ID = "find_test_item_by_id"
+    GET_TEST_ITEMS_FOR_LAUNCH = "get_test_items_for_launch"
+    GET_LOGS_FOR_TEST_ITEM = "get_logs_for_test_item"
+    GET_USER_INFORMATION = "get_user_information"
+    GET_DASHBOARD_DATA = "get_dashboard_data"

codemie_test_harness/tests/integrations/project/test_default_integrations.py

@@ -1,5 +1,3 @@
-import os
-
 import pytest
 
 from codemie_sdk.models.integration import (
@@ -11,8 +9,8 @@ from codemie_test_harness.tests.enums.tools import (
     NotificationTool,
     ProjectManagementTool,
     CodeBaseTool,
-    CloudTool,
     AccessManagementTool,
+    ReportPortalTool,
 )
 from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
     ado_test_plan_get_test_data,
@@ -36,11 +34,16 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     JIRA_TOOL_PROMPT,
     RESPONSE_FOR_JIRA_TOOL,
 )
+from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
+    rp_test_data,
+)
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.enums.environment import Environment
 from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.env_resolver import get_environment
 
 pytestmark = pytest.mark.skipif(
-    os.getenv("ENV") in ["local", "gcp"],
+    get_environment() in [Environment.LOCALHOST, Environment.GCP],
     reason="Skipping this test on local environment",
 )
 
@@ -55,12 +58,6 @@ pytestmark = pytest.mark.skipif(
 @pytest.mark.parametrize(
     "toolkit, tool_name, credential_type, credentials, prompt, expected_response",
     cloud_test_data,
-    ids=[
-        f"{Toolkit.CLOUD}_{CloudTool.AWS}",
-        f"{Toolkit.CLOUD}_{CloudTool.AZURE}",
-        f"{Toolkit.CLOUD}_{CloudTool.GCP}",
-        f"{Toolkit.CLOUD}_{CloudTool.KUBERNETES}",
-    ],
 )
 def test_assistant_with_default_integration_cloud(
     general_integration,
@@ -143,7 +140,6 @@ def test_assistant_with_default_integration_ado(
 @pytest.mark.parametrize(
     "toolkit, tool_name, credentials, prompt, expected_response",
     sonar_tools_test_data,
-    ids=[f"{row[0]}_{row[1]}" for row in sonar_tools_test_data],
 )
 def test_assistant_with_default_integration_codebase(
     general_integration,
@@ -314,7 +310,6 @@ def test_assistant_with_default_integration_email(
 @pytest.mark.project_integration
 @pytest.mark.default_integration
 @pytest.mark.regression
-@pytest.mark.regression
 def test_assistant_with_default_integration_keycloak(
     general_integration,
     integration_utils,
@@ -344,3 +339,43 @@ def test_assistant_with_default_integration_keycloak(
     response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
 
     similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
+
+
+@pytest.mark.assistant
+@pytest.mark.report_portal
+@pytest.mark.integration
+@pytest.mark.project_integration
+@pytest.mark.default_integration
+@pytest.mark.regression
+def test_assistant_with_default_integration_report_portal(
+    general_integration,
+    integration_utils,
+    assistant,
+    assistant_utils,
+    similarity_check,
+):
+    prompt = rp_test_data[7][2]
+    expected_response = rp_test_data[7][3]
+
+    #  delete existing integrations of the same type
+    for integration_type in IntegrationType:
+        integration_utils.delete_integrations_by_type(
+            integration_type, CredentialTypes.REPORT_PORTAL, test_project_name
+        )
+    # create a new integration
+    general_integration(
+        integration_type=IntegrationType.PROJECT,
+        credential_type=CredentialTypes.REPORT_PORTAL,
+        credential_values=CredentialsUtil.report_portal_credentials(),
+        project_name=test_project_name,
+    )
+    # create an assistant
+    report_portal_assistant = assistant(
+        Toolkit.REPORT_PORTAL,
+        ReportPortalTool.GET_DASHBOARD_DATA,
+        project_name=test_project_name,
+    )
+
+    response = assistant_utils.ask_assistant(report_portal_assistant, prompt)
+
+    similarity_check.check_similarity(response, expected_response)