codemie-test-harness 0.1.159__py3-none-any.whl → 0.1.161__py3-none-any.whl

codemie_test_harness/tests/test_data/file_management_tools_test_data.py

@@ -1,4 +1,4 @@
-import os
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 import pytest
 
@@ -89,29 +89,33 @@ GENERATE_IMAGE_TOOL_TASK = """
 """
 
 file_management_tools_test_data = [
-    (
+    pytest.param(
         FileManagementTool.PYTHON_CODE_INTERPRETER,
         CODE_INTERPRETER_TOOL_TASK,
         RESPONSE_FOR_CODE_INTERPRETER,
+        id=FileManagementTool.PYTHON_CODE_INTERPRETER,
     ),
     pytest.param(
         FileManagementTool.LIST_DIRECTORY,
         LIST_DIR_TOOL_TASK,
         RESPONSE_FOR_LIST_DIR,
         marks=pytest.mark.skipif(
-            os.getenv("ENV") == "local",
+            EnvironmentResolver.is_localhost(),
             reason="Skipping this test on local environment",
         ),
+        id=FileManagementTool.LIST_DIRECTORY,
     ),
-    (
+    pytest.param(
         FileManagementTool.WRITE_FILE,
         WRITE_FILE_TASK,
         RESPONSE_FOR_WRITE_FILE_TASK,
+        id=FileManagementTool.WRITE_FILE,
     ),
-    (
+    pytest.param(
         FileManagementTool.RUN_COMMAND_LINE,
         COMMAND_LINE_TOOL_TASK,
         RESPONSE_FOR_COMMAND_LINE_TASK,
+        id=FileManagementTool.RUN_COMMAND_LINE,
     ),
 ]
 

codemie_test_harness/tests/test_data/index_test_data.py

@@ -1,32 +1,30 @@
-import os
 from dataclasses import dataclass
 from typing import List
 
 import pytest
 
+from codemie_test_harness.tests.utils.env_resolver import get_environment
+from codemie_test_harness.tests.enums.environment import Environment
+
 
 @dataclass
 class EmbeddingData:
     """Data class to store Embedding models."""
 
     model_type: str
-    environments: List[str]
-
+    environments: List[Environment]
 
-AZURE_ENVS = ["preview", "azure", "local"]
-GCP_ENVS = ["preview", "gcp", "local"]
-AWS_ENVS = ["preview", "aws", "local"]
 
 MODELS = [
-    EmbeddingData("titan", AWS_ENVS),
-    EmbeddingData("gecko", GCP_ENVS),
-    EmbeddingData("ada-002", AZURE_ENVS),
+    EmbeddingData("titan", Environment.get_aws_environments()),
+    EmbeddingData("gecko", Environment.get_gcp_environments()),
+    EmbeddingData("ada-002", Environment.get_azure_environments()),
 ]
 
 
 def generate_test_data():
     """Generate pytest parameters for Embedding models"""
-    env = os.getenv("ENV")
+    env = get_environment()
     test_data = []
 
     for model in MODELS:
@@ -35,7 +33,7 @@ def generate_test_data():
                 model.model_type,
                 marks=pytest.mark.skipif(
                     env not in model.environments,
-                    reason=f"Skip on non {'/'.join(model.environments[:-1])} envs",
+                    reason=f"Skip on non {'/'.join(str(env) for env in model.environments[:-1])} envs",
                 ),
             )
         )

codemie_test_harness/tests/test_data/integrations_test_data.py

@@ -1,10 +1,9 @@
-import os
-
 import pytest
 
 from codemie_sdk.models.integration import CredentialTypes
 from codemie_test_harness.tests.enums.integrations import DataBaseDialect
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 valid_integrations = [
     pytest.param(
@@ -14,6 +13,7 @@ valid_integrations = [
             pytest.mark.aws,
             pytest.mark.cloud,
         ],
+        id=CredentialTypes.AWS,
     ),
     pytest.param(
         CredentialTypes.AZURE,
@@ -22,6 +22,7 @@ valid_integrations = [
             pytest.mark.azure,
             pytest.mark.cloud,
         ],
+        id=CredentialTypes.AZURE,
     ),
     pytest.param(
         CredentialTypes.GCP,
@@ -30,30 +31,35 @@ valid_integrations = [
             pytest.mark.gcp,
             pytest.mark.cloud,
             pytest.mark.skipif(
-                os.getenv("ENV") == "azure",
+                EnvironmentResolver.is_azure(),
                 reason="Still have an issue with encoding long strings",
             ),
         ],
+        id=CredentialTypes.GCP,
     ),
     pytest.param(
         CredentialTypes.SONAR,
         CredentialsUtil.sonar_credentials(),
         marks=pytest.mark.sonar,
+        id=f"{CredentialTypes.SONAR}_server",
     ),
     pytest.param(
         CredentialTypes.SONAR,
         CredentialsUtil.sonar_cloud_credentials(),
         marks=pytest.mark.sonar,
+        id=f"{CredentialTypes.SONAR}_cloud",
     ),
     pytest.param(
         CredentialTypes.GIT,
         CredentialsUtil.gitlab_credentials(),
         marks=pytest.mark.gitlab,
+        id=f"{CredentialTypes.GIT}_gitlab",
     ),
     pytest.param(
         CredentialTypes.GIT,
         CredentialsUtil.github_credentials(),
         marks=pytest.mark.github,
+        id=f"{CredentialTypes.GIT}_github",
     ),
     pytest.param(
         CredentialTypes.CONFLUENCE,
@@ -62,6 +68,7 @@ valid_integrations = [
             pytest.mark.confluence,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.CONFLUENCE}_server",
     ),
     pytest.param(
         CredentialTypes.CONFLUENCE,
@@ -71,6 +78,7 @@ valid_integrations = [
             pytest.mark.confluence_cloud,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.CONFLUENCE}_cloud",
     ),
     pytest.param(
         CredentialTypes.JIRA,
@@ -79,6 +87,7 @@ valid_integrations = [
             pytest.mark.jira,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.JIRA}_server",
     ),
     pytest.param(
         CredentialTypes.JIRA,
@@ -88,36 +97,43 @@ valid_integrations = [
             pytest.mark.jira_cloud,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.JIRA}_cloud",
     ),
     pytest.param(
         CredentialTypes.SQL,
         CredentialsUtil.sql_credentials(DataBaseDialect.POSTGRES),
         marks=pytest.mark.sql,
+        id=DataBaseDialect.POSTGRES,
     ),
     pytest.param(
         CredentialTypes.SQL,
         CredentialsUtil.sql_credentials(DataBaseDialect.MY_SQL),
         marks=pytest.mark.sql,
+        id=DataBaseDialect.MY_SQL,
     ),
     pytest.param(
         CredentialTypes.ELASTIC,
         CredentialsUtil.elastic_credentials(),
         marks=pytest.mark.elastic,
+        id=CredentialTypes.ELASTIC,
     ),
     pytest.param(
         CredentialTypes.MCP,
         CredentialsUtil.mcp_credentials(),
         marks=pytest.mark.mcp,
+        id=CredentialTypes.MCP,
     ),
     pytest.param(
         CredentialTypes.AZURE_DEVOPS,
         CredentialsUtil.azure_devops_credentials(),
         marks=pytest.mark.azure,
+        id=CredentialTypes.AZURE_DEVOPS,
     ),
     pytest.param(
         CredentialTypes.FILESYSTEM,
         CredentialsUtil.file_system_credentials(),
         marks=pytest.mark.file_system,
+        id=CredentialTypes.FILESYSTEM,
     ),
     pytest.param(
         CredentialTypes.EMAIL,
@@ -126,6 +142,7 @@ valid_integrations = [
             pytest.mark.notification,
             pytest.mark.email,
         ],
+        id=CredentialTypes.EMAIL,
     ),
     pytest.param(
         CredentialTypes.TELEGRAM,
@@ -134,16 +151,19 @@ valid_integrations = [
             pytest.mark.notification,
             pytest.mark.telegram,
         ],
+        id=CredentialTypes.TELEGRAM,
     ),
     pytest.param(
         CredentialTypes.SERVICE_NOW,
         CredentialsUtil.servicenow_credentials(),
         marks=pytest.mark.servicenow,
+        id=CredentialTypes.SERVICE_NOW,
     ),
     pytest.param(
         CredentialTypes.KEYCLOAK,
         CredentialsUtil.keycloak_credentials(),
         marks=pytest.mark.keycloak,
+        id=CredentialTypes.KEYCLOAK,
     ),
     pytest.param(
         CredentialTypes.KUBERNETES,
@@ -152,15 +172,17 @@ valid_integrations = [
             pytest.mark.kubernetes,
             pytest.mark.cloud,
             pytest.mark.skipif(
-                os.getenv("ENV") == "azure",
+                EnvironmentResolver.is_azure(),
                 reason="Still have an issue with encoding long strings",
             ),
         ],
+        id=CredentialTypes.KUBERNETES,
     ),
     pytest.param(
         CredentialTypes.REPORT_PORTAL,
         CredentialsUtil.report_portal_credentials(),
         marks=pytest.mark.report_portal,
+        id=CredentialTypes.REPORT_PORTAL,
     ),
 ]
 
@@ -172,6 +194,7 @@ testable_integrations = [
             pytest.mark.aws,
             pytest.mark.cloud,
         ],
+        id=CredentialTypes.AWS,
     ),
     pytest.param(
         CredentialTypes.AZURE,
@@ -180,6 +203,7 @@ testable_integrations = [
             pytest.mark.azure,
             pytest.mark.cloud,
         ],
+        id=CredentialTypes.AZURE,
     ),
     pytest.param(
         CredentialTypes.GCP,
@@ -188,20 +212,23 @@ testable_integrations = [
             pytest.mark.gcp,
             pytest.mark.cloud,
             pytest.mark.skipif(
-                os.getenv("ENV") == "azure",
+                EnvironmentResolver.is_azure(),
                 reason="Still have an issue with encoding long strings",
             ),
         ],
+        id=CredentialTypes.GCP,
     ),
     pytest.param(
         CredentialTypes.SONAR,
         CredentialsUtil.sonar_credentials(),
         marks=pytest.mark.sonar,
+        id=f"{CredentialTypes.SONAR}_server",
     ),
     pytest.param(
         CredentialTypes.SONAR,
         CredentialsUtil.sonar_cloud_credentials(),
         marks=pytest.mark.sonar,
+        id=f"{CredentialTypes.SONAR}_cloud",
     ),
     pytest.param(
         CredentialTypes.CONFLUENCE,
@@ -210,6 +237,7 @@ testable_integrations = [
             pytest.mark.confluence,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.CONFLUENCE}_server",
     ),
     pytest.param(
         CredentialTypes.CONFLUENCE,
@@ -219,6 +247,7 @@ testable_integrations = [
             pytest.mark.confluence_cloud,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.CONFLUENCE}_cloud",
     ),
     pytest.param(
         CredentialTypes.JIRA,
@@ -227,6 +256,7 @@ testable_integrations = [
             pytest.mark.jira,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.JIRA}_server",
     ),
     pytest.param(
         CredentialTypes.JIRA,
@@ -236,6 +266,7 @@ testable_integrations = [
             pytest.mark.jira_cloud,
             pytest.mark.project_management,
         ],
+        id=f"{CredentialTypes.JIRA}_cloud",
     ),
     pytest.param(
         CredentialTypes.EMAIL,
@@ -244,15 +275,17 @@ testable_integrations = [
             pytest.mark.email,
             pytest.mark.notification,
             pytest.mark.skipif(
-                os.getenv("ENV") == "local",
+                EnvironmentResolver.is_localhost(),
                 reason="Skipping this test on local environment",
             ),
         ],
+        id=CredentialTypes.EMAIL,
     ),
     pytest.param(
         CredentialTypes.SERVICE_NOW,
         CredentialsUtil.servicenow_credentials(),
         marks=pytest.mark.servicenow,
+        id=CredentialTypes.SERVICE_NOW,
     ),
     pytest.param(
         CredentialTypes.KUBERNETES,
@@ -261,15 +294,17 @@ testable_integrations = [
             pytest.mark.kubernetes,
             pytest.mark.cloud,
             pytest.mark.skipif(
-                os.getenv("ENV") == "azure",
+                EnvironmentResolver.is_azure(),
                 reason="Still have an issue with encoding long strings",
             ),
         ],
+        id=CredentialTypes.KUBERNETES,
     ),
     pytest.param(
         CredentialTypes.REPORT_PORTAL,
         CredentialsUtil.report_portal_credentials(),
         marks=pytest.mark.report_portal,
+        id=CredentialTypes.REPORT_PORTAL,
     ),
 ]
 
@@ -282,6 +317,7 @@ invalid_integrations = [
             pytest.mark.aws,
             pytest.mark.cloud,
         ],
+        id=CredentialTypes.AWS,
     ),
     pytest.param(
         CredentialTypes.AZURE,
@@ -291,6 +327,7 @@ invalid_integrations = [
             pytest.mark.azure,
             pytest.mark.cloud,
         ],
+        id=CredentialTypes.AZURE,
     ),
     pytest.param(
         CredentialTypes.GCP,
@@ -302,22 +339,25 @@ invalid_integrations = [
             pytest.mark.gcp,
             pytest.mark.cloud,
             pytest.mark.skipif(
-                os.getenv("ENV") == "azure",
+                EnvironmentResolver.is_azure(),
                 reason="Still have an issue with encoding long strings",
             ),
         ],
+        id=CredentialTypes.GCP,
     ),
     pytest.param(
         CredentialTypes.SONAR,
         CredentialsUtil.invalid_sonar_credentials(),
         "Invalid token",
         marks=pytest.mark.sonar,
+        id=f"{CredentialTypes.SONAR}_server",
     ),
     pytest.param(
         CredentialTypes.SONAR,
         CredentialsUtil.invalid_sonar_cloud_credentials(),
         "Invalid token",
         marks=pytest.mark.sonar,
+        id=f"{CredentialTypes.SONAR}_cloud",
     ),
     pytest.param(
         CredentialTypes.EMAIL,
@@ -327,28 +367,32 @@ invalid_integrations = [
             pytest.mark.email,
             pytest.mark.notification,
             pytest.mark.skipif(
-                os.getenv("ENV") == "local",
+                EnvironmentResolver.is_localhost(),
                 reason="Skipping this test on local environment",
             ),
         ],
+        id=CredentialTypes.EMAIL,
     ),
     pytest.param(
         CredentialTypes.JIRA,
         CredentialsUtil.invalid_jira_credentials(),
         "Unauthorized (401)",
         marks=pytest.mark.jira,
+        id=CredentialTypes.JIRA,
     ),
     pytest.param(
         CredentialTypes.CONFLUENCE,
         CredentialsUtil.invalid_confluence_credentials(),
         "Access denied",
         marks=pytest.mark.confluence,
+        id=CredentialTypes.CONFLUENCE,
     ),
     pytest.param(
         CredentialTypes.SERVICE_NOW,
         CredentialsUtil.invalid_servicenow_credentials(),
         'ServiceNow tool exception. Status: 401. Response: {"error":{"message":"User Not Authenticated","detail":"Required to provide Auth information"}',
         marks=pytest.mark.servicenow,
+        id=CredentialTypes.SERVICE_NOW,
     ),
     pytest.param(
         CredentialTypes.KUBERNETES,
@@ -358,11 +402,13 @@ invalid_integrations = [
             pytest.mark.kubernetes,
             pytest.mark.cloud,
         ],
+        id=CredentialTypes.KUBERNETES,
     ),
     pytest.param(
         CredentialTypes.REPORT_PORTAL,
         CredentialsUtil.invalid_report_portal_credentials(),
         "401 Client Error:  for url: https://report-portal.core.kuberocketci.io/api/v1/epm-cdme/launch?page.page=1",
         marks=pytest.mark.report_portal,
+        id=CredentialTypes.REPORT_PORTAL,
     ),
 ]

codemie_test_harness/tests/test_data/llm_test_data.py

@@ -3,6 +3,7 @@
 from dataclasses import dataclass
 from typing import List
 from codemie_test_harness.tests.enums.model_types import ModelTypes
+from codemie_test_harness.tests.enums.environment import Environment
 
 
 @dataclass
@@ -10,14 +11,14 @@ class LlmResponseData:
     """Data class to store LLM model response information."""
 
     model_type: ModelTypes
-    environments: List[str]
+    environments: List[Environment]
 
 
-# Define environment sets for common groups
-AZURE_ENVS = ["preview", "azure", "local"]
-GCP_ENVS = ["preview", "gcp", "local"]
-AWS_ENVS = ["preview", "aws", "local"]
-OTHER_ENVS = ["preview", "local"]
+# Define environment sets using centralized enum methods (type-safe enums)
+AZURE_ENVS = Environment.get_azure_environments()
+GCP_ENVS = Environment.get_gcp_environments()
+AWS_ENVS = Environment.get_aws_environments()
+OTHER_ENVS = [Environment.PREVIEW, Environment.LOCALHOST]
 
 # Define model responses with their environment restrictions
 MODEL_RESPONSES = [
@@ -52,6 +53,7 @@ MODEL_RESPONSES = [
     LlmResponseData(ModelTypes.CLAUDE_4_SONNET, AWS_ENVS),
     LlmResponseData(ModelTypes.CLAUDE_4_OPUS, AWS_ENVS),
     LlmResponseData(ModelTypes.CLAUDE_4_1_OPUS, AWS_ENVS),
+    LlmResponseData(ModelTypes.CLAUDE_4_SONNET_1M, [Environment.AWS]),
     # Other LLMs test data
     LlmResponseData(ModelTypes.RLAB_QWQ_32B, OTHER_ENVS),
     LlmResponseData(ModelTypes.DEEPSEEK_R1, OTHER_ENVS),

codemie_test_harness/tests/test_data/project_management_test_data.py

@@ -20,6 +20,7 @@ pm_tools_test_data = [
         JIRA_TOOL_PROMPT,
         RESPONSE_FOR_JIRA_TOOL,
         marks=pytest.mark.jira,
+        id=ProjectManagementIntegrationType.JIRA,
     ),
     pytest.param(
         ProjectManagementTool.CONFLUENCE,
@@ -27,6 +28,7 @@ pm_tools_test_data = [
         CONFLUENCE_TOOL_PROMPT,
         RESPONSE_FOR_CONFLUENCE_TOOL,
         marks=pytest.mark.confluence,
+        id=ProjectManagementIntegrationType.CONFLUENCE,
     ),
     pytest.param(
         ProjectManagementTool.JIRA,
@@ -34,6 +36,7 @@ pm_tools_test_data = [
         JIRA_CLOUD_TOOL_PROMPT,
         RESPONSE_FOR_JIRA_CLOUD_TOOL,
         marks=[pytest.mark.jira, pytest.mark.jira_cloud],
+        id=ProjectManagementIntegrationType.JIRA_CLOUD,
     ),
     pytest.param(
         ProjectManagementTool.CONFLUENCE,
@@ -41,5 +44,6 @@ pm_tools_test_data = [
         CONFLUENCE_CLOUD_TOOL_PROMPT,
         RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL,
         marks=[pytest.mark.confluence, pytest.mark.confluence_cloud],
+        id=ProjectManagementIntegrationType.CONFLUENCE_CLOUD,
     ),
 ]

codemie_test_harness/tests/test_data/vcs_tools_test_data.py

@@ -2,6 +2,7 @@ import os
 
 import pytest
 
+from codemie_sdk.models.integration import CredentialTypes
 from codemie_test_harness.tests.enums.tools import VcsTool
 
 GITHUB_TOOL_TASK = (
@@ -60,9 +61,17 @@ RESPONSE_FOR_GITLAB = f"""
 
 vcs_tools_test_data = [
     pytest.param(
-        VcsTool.GITHUB, GITHUB_TOOL_TASK, RESPONSE_FOR_GITHUB, marks=pytest.mark.github
+        VcsTool.GITHUB,
+        GITHUB_TOOL_TASK,
+        RESPONSE_FOR_GITHUB,
+        marks=pytest.mark.github,
+        id=f"{CredentialTypes.GIT}_github",
     ),
     pytest.param(
-        VcsTool.GITLAB, GITLAB_TOOL_TASK, RESPONSE_FOR_GITLAB, marks=pytest.mark.gitlab
+        VcsTool.GITLAB,
+        GITLAB_TOOL_TASK,
+        RESPONSE_FOR_GITLAB,
+        marks=pytest.mark.gitlab,
+        id=f"{CredentialTypes.GIT}_gitlab",
     ),
 ]

codemie_test_harness/tests/utils/aws_parameters_store.py

@@ -11,6 +11,7 @@ from botocore.exceptions import ClientError
 
 from codemie_sdk.models.integration import CredentialValues
 from codemie_test_harness.tests.enums.integrations import DataBaseDialect
+from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
 class AwsParameterStore:
@@ -419,7 +420,7 @@ class CredentialsUtil:
         elastic_creds = AwsParameterStore.get_cloud_provider_credentials("elastic")
         elastic_creds = (
             elastic_creds.get("elasticsearch", {})
-            if os.getenv("ENV") == "preview"
+            if EnvironmentResolver.is_preview()
             else elastic_creds.get("sandbox", {}).get("elasticsearch", {})
         )
         return [
@@ -434,7 +435,9 @@ class CredentialsUtil:
     def sql_credentials(db_dialect: DataBaseDialect) -> List[CredentialValues]:
         sql_creds = AwsParameterStore.get_cloud_provider_credentials("sql")
         sql_creds = (
-            sql_creds if os.getenv("ENV") == "preview" else sql_creds.get("sandbox", {})
+            sql_creds
+            if EnvironmentResolver.is_preview()
+            else sql_creds.get("sandbox", {})
         )
 
         return [
@@ -620,6 +623,24 @@ class CredentialsUtil:
                 cred.value = "wrong_key"
         return credentials
 
+    @staticmethod
+    def lite_llm_credentials() -> List[CredentialValues]:
+        lite_llm_creds = AwsParameterStore.get_cloud_provider_credentials("litellm")
+        return [
+            CredentialValues(
+                key="api_key",
+                value=lite_llm_creds.get("api_key"),
+            ),
+        ]
+
+    @staticmethod
+    def invalid_lite_llm_credentials() -> List[CredentialValues]:
+        credentials = CredentialsUtil.lite_llm_credentials()
+        for cred in credentials:
+            if cred.key == "api_key":
+                cred.value = "wrong_key"
+        return credentials
+
     @staticmethod
     def jira_cloud_jql() -> str:
         jira_creds = AwsParameterStore.get_cloud_provider_credentials("jira")

codemie_test_harness/tests/utils/constants.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from codemie_test_harness.tests.enums.tools import VcsTool, NotificationTool
 
 
-class ProjectManagementIntegrationType(Enum):
+class ProjectManagementIntegrationType(str, Enum):
     JIRA = "jira"
     CONFLUENCE = "confluence"
     JIRA_CLOUD = "jira_cloud"