codemie-test-harness 0.1.133__tar.gz → 0.1.135__tar.gz

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.133
+Version: 0.1.135
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.133)
+Requires-Dist: codemie-sdk-python (==0.1.135)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py

@@ -1,6 +1,7 @@
 """Tests for Confluence datasource operations - Final version."""
 
 import os
+
 import pytest
 from hamcrest import assert_that, equal_to
 from requests import HTTPError
@@ -8,31 +9,60 @@ from requests import HTTPError
 from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     CONFLUENCE_TOOL_PROMPT,
     RESPONSE_FOR_CONFLUENCE_TOOL,
+    CONFLUENCE_CLOUD_TOOL_PROMPT,
+    RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL,
 )
+from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
 from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
 
 
+@pytest.fixture(scope="function")
+def confluence_cloud_datasource(datasource_utils, confluence_cloud_integration):
+    datasource = datasource_utils.create_confluence_datasource(
+        setting_id=confluence_cloud_integration.id,
+        cql=CredentialsUtil.confluence_cloud_jql(),
+    )
+    yield datasource
+    if datasource:
+        datasource_utils.delete_datasource(datasource)
+
+
 @pytest.mark.regression
-def test_create_datasource_with_confluence_integration(
+@pytest.mark.parametrize(
+    "datasource_fixture, prompt, expected_response",
+    [
+        ("confluence_datasource", CONFLUENCE_TOOL_PROMPT, RESPONSE_FOR_CONFLUENCE_TOOL),
+        (
+            "confluence_cloud_datasource",
+            CONFLUENCE_CLOUD_TOOL_PROMPT,
+            RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL,
+        ),
+    ],
+    ids=["Confluence", "Confluence Cloud"],
+)
+def test_create_datasource_with_confluence_and_confluence_cloud_integration(
+    request,
     assistant,
     assistant_utils,
     datasource_utils,
-    confluence_datasource,
     kb_context,
-    default_llm,
     similarity_check,
+    datasource_fixture,
+    prompt,
+    expected_response,
 ):
-    assistant = assistant(context=kb_context(confluence_datasource))
+    datasource = request.getfixturevalue(datasource_fixture)
+    assistant = assistant(context=kb_context(datasource))
 
-    response = assistant_utils.ask_assistant(assistant, CONFLUENCE_TOOL_PROMPT)
-    similarity_check.check_similarity(response, RESPONSE_FOR_CONFLUENCE_TOOL)
+    response = assistant_utils.ask_assistant(assistant, prompt)
+    similarity_check.check_similarity(response, expected_response)
 
     datasource_utils.update_confluence_datasource(
-        confluence_datasource.id, full_reindex=True, skip_reindex=False
+        datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response = assistant_utils.ask_assistant(assistant, CONFLUENCE_TOOL_PROMPT)
-    similarity_check.check_similarity(response, RESPONSE_FOR_CONFLUENCE_TOOL)
+    response = assistant_utils.ask_assistant(assistant, prompt)
+    similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.regression
@@ -51,7 +81,7 @@ def test_edit_description_for_confluence_data_source(
 
 @pytest.mark.regression
 def test_create_confluence_datasource_with_existing_name(
-    confluence_datasource, datasource_utils
+    datasource_utils, confluence_datasource
 ):
     datasource = datasource_utils.get_datasource(confluence_datasource.id)
 

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py

@@ -8,36 +8,60 @@ from requests import HTTPError
 from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     JIRA_TOOL_PROMPT,
     RESPONSE_FOR_JIRA_TOOL,
+    JIRA_CLOUD_TOOL_PROMPT,
+    RESPONSE_FOR_JIRA_CLOUD_TOOL,
 )
+from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
 from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
 
 
+@pytest.fixture(scope="function")
+def jira_cloud_datasource(datasource_utils, jira_cloud_integration):
+    datasource = datasource_utils.create_jira_datasource(
+        setting_id=jira_cloud_integration.id, jql=CredentialsUtil.jira_cloud_jql()
+    )
+    yield datasource
+    if datasource:
+        datasource_utils.delete_datasource(datasource)
+
+
 @pytest.mark.regression
-def test_create_datasource_with_jira_integration(
+@pytest.mark.parametrize(
+    "datasource_fixture, prompt, expected_response",
+    [
+        ("jira_datasource", JIRA_TOOL_PROMPT, RESPONSE_FOR_JIRA_TOOL),
+        ("jira_cloud_datasource", JIRA_CLOUD_TOOL_PROMPT, RESPONSE_FOR_JIRA_CLOUD_TOOL),
+    ],
+    ids=["Jira", "Jira Cloud"],
+)
+def test_create_datasource_with_jira_and_jira_cloud_integration(
+    request,
     assistant,
     assistant_utils,
     datasource_utils,
-    jira_datasource,
     kb_context,
-    default_llm,
     similarity_check,
+    datasource_fixture,
+    prompt,
+    expected_response,
 ):
-    assistant = assistant(context=kb_context(jira_datasource))
+    datasource = request.getfixturevalue(datasource_fixture)
+    assistant = assistant(context=kb_context(datasource))
 
-    response = assistant_utils.ask_assistant(assistant, JIRA_TOOL_PROMPT)
-    similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
+    response = assistant_utils.ask_assistant(assistant, prompt)
+    similarity_check.check_similarity(response, response)
 
     datasource_utils.update_jira_datasource(
-        jira_datasource.id, full_reindex=True, skip_reindex=False
+        datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response = assistant_utils.ask_assistant(assistant, JIRA_TOOL_PROMPT)
-    similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
+    response = assistant_utils.ask_assistant(assistant, prompt)
+    similarity_check.check_similarity(response, response)
 
 
 @pytest.mark.regression
 def test_edit_description_for_jira_data_source(
-    client, jira_datasource, datasource_utils
+    client, datasource_utils, jira_datasource
 ):
     updated_description = get_random_name()
 
@@ -50,7 +74,7 @@ def test_edit_description_for_jira_data_source(
 
 
 @pytest.mark.regression
-def test_create_jira_datasource_with_existing_name(jira_datasource, datasource_utils):
+def test_create_jira_datasource_with_existing_name(datasource_utils, jira_datasource):
     datasource = datasource_utils.get_datasource(jira_datasource.id)
 
     try:

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/codemie_test_harness/tests/enums/integrations.py

@@ -4,5 +4,6 @@ from enum import Enum
 class DataBaseDialect(str, Enum):
     """Enum for DB Dialect names."""
 
+    MS_SQL = "mssql"
     MY_SQL = "mysql"
     POSTGRES = "postgres"

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/codemie_test_harness/tests/test_data/data_management_tools_test_data.py

@@ -1,3 +1,7 @@
+import os
+
+import pytest
+
 from codemie_test_harness.tests.enums.integrations import DataBaseDialect
 
 ELASTIC_TOOL_TASK = """
@@ -81,7 +85,15 @@ RESPONSE_FOR_SQL = """
     | Sarah      | Connor    | sarah.connor@email.com  | Security   |
 """
 
+# Define test data for SQL tools based on environment
 sql_tools_test_data = [
     DataBaseDialect.MY_SQL,
     DataBaseDialect.POSTGRES,
+    pytest.param(
+        DataBaseDialect.MS_SQL,
+        marks=pytest.mark.skipif(
+            os.getenv("ENV") not in ("aws", "azure", "gcp"),
+            reason="MS SQL is only available in staging environments",
+        ),
+    ),
 ]

codemie_test_harness-0.1.135/codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py

@@ -0,0 +1,48 @@
+import os
+
+import pytest
+
+from codemie_test_harness.tests.enums.tools import DataManagementTool, Toolkit
+from codemie_test_harness.tests.enums.integrations import DataBaseDialect
+
+sql_tools_test_data = [
+    (
+        Toolkit.DATA_MANAGEMENT,
+        DataManagementTool.SQL,
+        DataBaseDialect.MY_SQL,
+        {"sql_query": "SHOW TABLES"},
+        [{"Tables_in_my_database": "products"}, {"Tables_in_my_database": "users"}],
+    ),
+    (
+        Toolkit.DATA_MANAGEMENT,
+        DataManagementTool.SQL,
+        DataBaseDialect.POSTGRES,
+        {
+            "sql_query": "SELECT table_name FROM information_schema.tables WHERE table_schema='public';"
+        },
+        [{"table_name": "users"}, {"table_name": "products"}],
+    ),
+    pytest.param(
+        Toolkit.DATA_MANAGEMENT,
+        DataManagementTool.SQL,
+        DataBaseDialect.MS_SQL,
+        {
+            "sql_query": """SELECT table_name
+                            FROM
+                            information_schema.tables
+                            WHERE
+                            table_type = 'BASE TABLE'
+                            AND
+                            table_catalog = 'autotests'
+                            AND
+                            table_schema = 'dbo';
+                            """
+        },
+        [{"table_name": "Users"}, {"table_name": "Products"}],
+        marks=pytest.mark.skipif(
+            os.getenv("ENV") not in ("aws", "azure", "gcp"),
+            reason="MS SQL is only available in staging environments",
+        ),
+        id="DataBaseDialect.MS_SQL",
+    ),
+]

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/codemie_test_harness/tests/test_data/pm_tools_test_data.py

@@ -72,10 +72,20 @@ CONFLUENCE_CLOUD_TOOL_PROMPT = """
     relative_url='/rest/api/content/search'
     method='GET'
     params='{'cql': 'title ~ "codemie"', 'limit': 5, 'expand': 'body.storage'}'
+    or Search Kb tool
+    and return the page content
 """
 
 RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL = """
-    I found one page related to "codemie":
-    - **Title:** [Codemie Platform](https://codemie-test.atlassian.net/wiki/spaces/~71202043303b83370a486ea6ba5a1da8884d63/pages/131223/Codemie+Platform)
-    Let me know if you need more information regarding this page or anything else.
+    Here are some pages related to "codemie" from the Confluence space:
+    Codemie Platform
+    Title: Codemie Platform
+    URL: Codemie Platform
+    Description: This page mentions that "Codemie Platform is the best one."
+
+    CODEMIE Overview
+    Title: CODEMIE
+    URL: CODEMIE Overview
+    Description: This space includes a section for describing the purpose of the space, a search feature, and a filter by label functionality.
+    If you need more detailed information about a specific page, please let me know!
 """

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/codemie_test_harness/tests/utils/aws_parameters_store.py

@@ -563,3 +563,15 @@ class CredentialsUtil:
             if cred.key == "kubernetes_token":
                 cred.value = "wrong_token"
         return credentials
+
+    @staticmethod
+    def jira_cloud_jql() -> str:
+        jira_creds = AwsParameterStore.get_cloud_provider_credentials("jira")
+        return jira_creds.get("jiracloud", {}).get("jql")
+
+    @staticmethod
+    def confluence_cloud_jql() -> str:
+        confluence_creds = AwsParameterStore.get_cloud_provider_credentials(
+            "confluence"
+        )
+        return confluence_creds.get("confluencecloud", {}).get("cql")

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools.py

@@ -22,6 +22,7 @@ pytestmark = pytest.mark.skipif(
 @pytest.mark.parametrize(
     "toolkit,tool_name,db_dialect,prompt,expected_response",
     sql_tools_test_data,
+    ids=[f"{row[2]}" for row in sql_tools_test_data],
 )
 def test_workflow_with_sql_tools_direct(
     workflow_with_tool,
@@ -59,6 +60,7 @@ def test_workflow_with_sql_tools_direct(
 @pytest.mark.parametrize(
     "toolkit,tool_name,db_dialect,prompt,expected_response",
     sql_tools_test_data,
+    ids=[f"{row[2]}" for row in sql_tools_test_data],
 )
 def test_workflow_with_sql_tools_with_hardcoded_args(
     workflow_with_tool,
@@ -97,6 +99,7 @@ def test_workflow_with_sql_tools_with_hardcoded_args(
 @pytest.mark.parametrize(
     "toolkit,tool_name,db_dialect,prompt,expected_response",
     sql_tools_test_data,
+    ids=[f"{row[2]}" for row in sql_tools_test_data],
 )
 def test_workflow_with_sql_tools_with_overriding_args(
     workflow_with_tool,

{codemie_test_harness-0.1.133 → codemie_test_harness-0.1.135}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemie-test-harness"
-version = "0.1.133"
+version = "0.1.135"
 description = "Autotest for CodeMie backend and UI"
 authors = [
     "Anton Yeromin <anton_yeromin@epam.com>",
@@ -12,7 +12,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.12"
-codemie-sdk-python = "0.1.133"
+codemie-sdk-python = "0.1.135"
 pytest = "^8.4.1"
 python-dotenv = "^1.1.0"
 PyHamcrest = "^2.1.0"

codemie_test_harness-0.1.133/codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py

@@ -1,21 +0,0 @@
-from codemie_test_harness.tests.enums.tools import DataManagementTool, Toolkit
-from codemie_test_harness.tests.enums.integrations import DataBaseDialect
-
-sql_tools_test_data = [
-    (
-        Toolkit.DATA_MANAGEMENT,
-        DataManagementTool.SQL,
-        DataBaseDialect.MY_SQL,
-        {"sql_query": "SHOW TABLES"},
-        [{"Tables_in_my_database": "products"}, {"Tables_in_my_database": "users"}],
-    ),
-    (
-        Toolkit.DATA_MANAGEMENT,
-        DataManagementTool.SQL,
-        DataBaseDialect.POSTGRES,
-        {
-            "sql_query": "SELECT table_name FROM information_schema.tables WHERE table_schema='public';"
-        },
-        [{"table_name": "users"}, {"table_name": "products"}],
-    ),
-]