codemie-test-harness 0.1.157__tar.gz → 0.1.159__tar.gz

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.157
+Version: 0.1.159
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.157)
+Requires-Dist: codemie-sdk-python (==0.1.159)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py

@@ -21,12 +21,15 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
 )
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
 
+pytestmark = pytest.mark.skipif(
+    os.getenv("ENV") == "local", reason="Skipping this tests on local environment"
+)
+
 
 @pytest.mark.assistant
 @pytest.mark.elastic
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-6132")
-@pytest.mark.skip(reason="Credentials for Elastic are not available")
 def test_create_assistant_with_elastic_tool(
     integration_utils, assistant, assistant_utils, similarity_check
 ):
@@ -49,14 +52,15 @@ def test_create_assistant_with_elastic_tool(
 @pytest.mark.sql
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-6132")
-@pytest.mark.skipif(
-    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
-)
 @pytest.mark.parametrize(
     "db_dialect",
     sql_tools_test_data,
     ids=[DataBaseDialect.MY_SQL, DataBaseDialect.POSTGRES, DataBaseDialect.MS_SQL],
 )
+@pytest.mark.testcase("EPMCDME-6132")
+@pytest.mark.skipif(
+    os.getenv("ENV") == "local", reason="Skipping this test on local environment"
+)
 def test_create_assistant_with_sql_tool(
     integration_utils, assistant_utils, assistant, similarity_check, db_dialect
 ):

codemie_test_harness-0.1.159/codemie_test_harness/tests/assistant/tools/report_portal/test_assistant_report_portal_tools.py

@@ -0,0 +1,32 @@
+import pytest
+
+from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
+    rp_test_data,
+)
+
+
+@pytest.mark.assistant
+@pytest.mark.report_portal
+@pytest.mark.regression
+@pytest.mark.parametrize(
+    "toolkit,tool_name,prompt,expected_response",
+    rp_test_data,
+    ids=[f"{row[0]}_{row[1]}" for row in rp_test_data],
+)
+def test_assistant_with_report_portal_tools(
+    assistant,
+    assistant_utils,
+    report_portal_integration,
+    similarity_check,
+    toolkit,
+    tool_name,
+    prompt,
+    expected_response,
+):
+    assistant = assistant(
+        toolkit,
+        tool_name,
+        settings=report_portal_integration,
+    )
+    response = assistant_utils.ask_assistant(assistant, prompt)
+    similarity_check.check_similarity(response, expected_response)

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/conftest.py

@@ -374,6 +374,17 @@ def keycloak_integration(integration_utils):
         integration_utils.delete_integration(integration)
 
 
+@pytest.fixture(scope="module")
+def report_portal_integration(integration_utils):
+    credential_values = CredentialsUtil.report_portal_credentials()
+    integration = integration_utils.create_integration(
+        CredentialTypes.REPORT_PORTAL, credential_values
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
 @pytest.fixture(scope="function")
 def general_integration(integration_utils):
     created_integration: Optional[Integration] = None

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/enums/tools.py

@@ -20,6 +20,7 @@ class Toolkit(str, Enum):
     DATA_MANAGEMENT = "Data Management"
     SERVICENOW = "IT Service Management"
     ACCESS_MANAGEMENT = "Access Management"
+    REPORT_PORTAL = "Report Portal"
 
 
 class GitTool(str, Enum):
@@ -185,3 +186,16 @@ class AccessManagementTool(str, Enum):
     """Enum for Access Management tool names."""
 
     KEYCLOAK = "keycloak"
+
+
+class ReportPortalTool(str, Enum):
+    """Enum for Report Portal tool names."""
+
+    GET_EXTENDED_LAUNCH_DATA_AS_RAW = "get_extended_launch_data_as_raw"
+    GET_LAUNCH_DETAILS = "get_launch_details"
+    GET_ALL_LAUNCHES = "get_all_launches"
+    FIND_TEST_ITEM_BY_ID = "find_test_item_by_id"
+    GET_TEST_ITEMS_FOR_LAUNCH = "get_test_items_for_launch"
+    GET_LOGS_FOR_TEST_ITEM = "get_logs_for_test_item"
+    GET_USER_INFORMATION = "get_user_information"
+    GET_DASHBOARD_DATA = "get_dashboard_data"

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/integrations/project/test_default_integrations.py

@@ -13,6 +13,7 @@ from codemie_test_harness.tests.enums.tools import (
     CodeBaseTool,
     CloudTool,
     AccessManagementTool,
+    ReportPortalTool,
 )
 from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
     ado_test_plan_get_test_data,
@@ -36,6 +37,9 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     JIRA_TOOL_PROMPT,
     RESPONSE_FOR_JIRA_TOOL,
 )
+from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
+    rp_test_data,
+)
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
 from codemie_test_harness.tests.utils.constants import test_project_name
 
@@ -314,7 +318,6 @@ def test_assistant_with_default_integration_email(
 @pytest.mark.project_integration
 @pytest.mark.default_integration
 @pytest.mark.regression
-@pytest.mark.regression
 def test_assistant_with_default_integration_keycloak(
     general_integration,
     integration_utils,
@@ -344,3 +347,43 @@ def test_assistant_with_default_integration_keycloak(
     response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
 
     similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
+
+
+@pytest.mark.assistant
+@pytest.mark.report_portal
+@pytest.mark.integration
+@pytest.mark.project_integration
+@pytest.mark.default_integration
+@pytest.mark.regression
+def test_assistant_with_default_integration_report_portal(
+    general_integration,
+    integration_utils,
+    assistant,
+    assistant_utils,
+    similarity_check,
+):
+    prompt = rp_test_data[7][2]
+    expected_response = rp_test_data[7][3]
+
+    #  delete existing integrations of the same type
+    for integration_type in IntegrationType:
+        integration_utils.delete_integrations_by_type(
+            integration_type, CredentialTypes.REPORT_PORTAL, test_project_name
+        )
+    # create a new integration
+    general_integration(
+        integration_type=IntegrationType.PROJECT,
+        credential_type=CredentialTypes.REPORT_PORTAL,
+        credential_values=CredentialsUtil.report_portal_credentials(),
+        project_name=test_project_name,
+    )
+    # create an assistant
+    report_portal_assistant = assistant(
+        Toolkit.REPORT_PORTAL,
+        ReportPortalTool.GET_DASHBOARD_DATA,
+        project_name=test_project_name,
+    )
+
+    response = assistant_utils.ask_assistant(report_portal_assistant, prompt)
+
+    similarity_check.check_similarity(response, expected_response)

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/integrations/project/test_project_integrations.py

@@ -50,6 +50,7 @@ from codemie_test_harness.tests.utils.base_utils import (
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
         f"Project integration: {CredentialTypes.KEYCLOAK}",
+        f"Project integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_created(
@@ -96,6 +97,7 @@ def test_integration_created(
         f"Project integration: {CredentialTypes.EMAIL}",
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
+        f"Project integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_after_creation(
@@ -158,6 +160,7 @@ def test_integration_after_creation(
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
         f"Project integration: {CredentialTypes.KEYCLOAK}",
+        f"Project integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_update_integration(
@@ -214,6 +217,7 @@ def test_update_integration(
         f"Project integration: {CredentialTypes.CONFLUENCE}",
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
+        f"Project integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_with_invalid_credentials(
@@ -269,6 +273,7 @@ def test_integration_with_invalid_credentials(
         f"Project integration: {CredentialTypes.EMAIL}",
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
+        f"Project integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_during_creation(credential_type, credentials, integration_utils):
@@ -305,6 +310,7 @@ def test_integration_during_creation(credential_type, credentials, integration_u
         f"Project integration: {CredentialTypes.CONFLUENCE}",
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
+        f"Project integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_during_creation_with_invalid_credentials(
@@ -352,6 +358,7 @@ def test_integration_during_creation_with_invalid_credentials(
         f"Project integration: {CredentialTypes.SERVICE_NOW}",
         f"Project integration: {CredentialTypes.KUBERNETES}",
         f"Project integration: {CredentialTypes.KEYCLOAK}",
+        f"Project integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_delete_integration(

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/integrations/user/test_default_integrations.py

@@ -13,6 +13,7 @@ from codemie_test_harness.tests.enums.tools import (
     CodeBaseTool,
     CloudTool,
     AccessManagementTool,
+    ReportPortalTool,
 )
 from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
     ado_test_plan_get_test_data,
@@ -36,6 +37,9 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     JIRA_TOOL_PROMPT,
     RESPONSE_FOR_JIRA_TOOL,
 )
+from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
+    rp_test_data,
+)
 from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
 from codemie_test_harness.tests.utils.constants import test_project_name
 
@@ -343,3 +347,43 @@ def test_assistant_with_default_integration_keycloak(
     response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
 
     similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
+
+
+@pytest.mark.assistant
+@pytest.mark.report_portal
+@pytest.mark.integration
+@pytest.mark.user_integration
+@pytest.mark.default_integration
+@pytest.mark.regression
+def test_assistant_with_default_integration_report_portal(
+    general_integration,
+    integration_utils,
+    assistant,
+    assistant_utils,
+    similarity_check,
+):
+    prompt = rp_test_data[7][2]
+    expected_response = rp_test_data[7][3]
+
+    #  delete existing integrations of the same type
+    for integration_type in IntegrationType:
+        integration_utils.delete_integrations_by_type(
+            integration_type, CredentialTypes.REPORT_PORTAL, test_project_name
+        )
+    # create a new integration
+    general_integration(
+        integration_type=IntegrationType.USER,
+        credential_type=CredentialTypes.REPORT_PORTAL,
+        credential_values=CredentialsUtil.report_portal_credentials(),
+        project_name=test_project_name,
+    )
+    # create an assistant
+    report_portal_assistant = assistant(
+        Toolkit.REPORT_PORTAL,
+        ReportPortalTool.GET_DASHBOARD_DATA,
+        project_name=test_project_name,
+    )
+
+    response = assistant_utils.ask_assistant(report_portal_assistant, prompt)
+
+    similarity_check.check_similarity(response, expected_response)

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/integrations/user/test_user_integrations.py

@@ -51,6 +51,7 @@ from codemie_test_harness.tests.utils.base_utils import (
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
         f"User integration: {CredentialTypes.KEYCLOAK}",
+        f"User integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_created(
@@ -98,6 +99,7 @@ def test_integration_created(
         f"User integration: {CredentialTypes.EMAIL}",
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
+        f"User integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_after_creation(
@@ -160,6 +162,7 @@ def test_integration_after_creation(
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
         f"User integration: {CredentialTypes.KEYCLOAK}",
+        f"User integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_update_integration(
@@ -217,6 +220,7 @@ def test_update_integration(
         f"User integration: {CredentialTypes.CONFLUENCE}",
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
+        f"User integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_with_invalid_credentials(
@@ -272,6 +276,7 @@ def test_integration_with_invalid_credentials(
         f"User integration: {CredentialTypes.EMAIL}",
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
+        f"User integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_during_creation(credential_type, credentials, integration_utils):
@@ -308,6 +313,7 @@ def test_integration_during_creation(credential_type, credentials, integration_u
         f"User integration: {CredentialTypes.CONFLUENCE}",
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
+        f"User integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_integration_during_creation_with_invalid_credentials(
@@ -355,6 +361,7 @@ def test_integration_during_creation_with_invalid_credentials(
         f"User integration: {CredentialTypes.SERVICE_NOW}",
         f"User integration: {CredentialTypes.KUBERNETES}",
         f"User integration: {CredentialTypes.KEYCLOAK}",
+        f"User integration: {CredentialTypes.REPORT_PORTAL}",
     ],
 )
 def test_delete_integration(

{codemie_test_harness-0.1.157 → codemie_test_harness-0.1.159}/codemie_test_harness/tests/test_data/data_management_tools_test_data.py

@@ -5,46 +5,13 @@ import pytest
 from codemie_test_harness.tests.enums.integrations import DataBaseDialect
 
 ELASTIC_TOOL_TASK = """
-    Search for Document ID: Zx4eF5cBvnDA_Y45jlWN in codemie_metrics_logs index
+    send the query to Elastic:
+    {'index': 'codemie-*', 'query': {'size': 0, 'aggs': {'indices': {'terms': {'field': '_index', 'size': 5, 'order': {'_count': 'asc'}}}}}}
+    return only the number of failed from _shards node.
 """
 
 RESPONSE_FOR_ELASTIC = """
-    Here are the details of the document with ID `Zx4eF5cBvnDA_Y45jlWN` in the index `codemie_metrics_logs`:
-
-    - **Timestamp:** 2025-05-28T13:39:27.949Z
-    - **Metric Name:** conversation_assistant_usage
-    - **Attributes:**
-      - Count: 1
-      - User ID: 6c04c3d1-7967-481a-82d5-0b7da258a555
-      - User Name: Andrei Maskouchanka
-      - Assistant ID: 4657258e-19e5-4162-a6ec-2a76d11c14d1
-      - Assistant Name: Elastic Tool
-      - Input Tokens: 2414
-      - Output Tokens: 434
-      - Money Spent: 0.01858
-      - Project: andrei_maskouchanka@epam.com
-      - Execution Time: 11.09025979042053
-      - LLM Model: gpt-4o
-      - Conversation ID: 9ea412d5-c460-41f1-8071-35083538652c
-      - Status: success
-    
-    - **Additional Information:**
-      - Time: 2025-05-28T13:39:27.949952597Z
-      - Stream: stderr
-      - Level: INFO
-      - UUID: 3fd6a83f6a266c7760a71406dbbef99f
-      - Kubernetes Details:
-        - Pod Name: codemie-api-69b9484d46-2hnpd
-        - Namespace Name: preview-codemie
-        - Pod ID: 97edb665-2b76-47ad-a545-3b7a2695a0e7
-        - Host: gke-epmd-edp-anthos--nap-e2-standard--e45b29fc-863z
-        - Pod IP: 172.20.6.90
-        - Container Name: codemie
-        - Docker ID: 11ec33dc943cfa7153f31cde412837ccfae711b1edc9bb7d20055614eeefbf15
-        - Container Hash: europe-west3-docker.pkg.dev/or2-msq-epmd-edp-anthos-t1iylu/prod/code-assistant@sha256:a0271b31b2a55cf71d319c6b5f394fbe7c6833db74fbb51ab22680bf096c6ecb
-        - Container Image: europe-west3-docker.pkg.dev/or2-msq-epmd-edp-anthos-t1iylu/prod/code-assistant:0.28.0-SNAPSHOT.10
-    
-    If you need further details or assistance, please let me know!    
+    The query was executed successfully. The number of failed shards from the _shards node is 0.    
 """
 
 SQL_TOOL_CREATE_TABLE_TASK = """