codemie-test-harness 0.1.204__tar.gz → 0.1.206__tar.gz

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.204
+Version: 0.1.206
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.204)
+Requires-Dist: codemie-sdk-python (==0.1.206)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/pytest.ini

@@ -1,7 +1,6 @@
 [pytest]
 addopts = -v
 timeout = 300
-timeout_method = signal
 timeout_func_only = true
 filterwarnings =
     ignore::pytest.PytestUnknownMarkWarning

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/assistant/datasource/test_code_datasource.py

@@ -22,7 +22,7 @@ from codemie_test_harness.tests.utils.base_utils import (
     ids=[f"{row[0][0]}" for row in index_test_data],
 )
 def test_create_index_application_with_embedding_model(
-    git_integration,
+    gitlab_integration,
     datasource_utils,
     assistant,
     assistant_utils,
@@ -37,7 +37,7 @@ def test_create_index_application_with_embedding_model(
     """
 
     datasource = datasource_utils.create_gitlab_datasource(
-        setting_id=git_integration.id, embeddings_model=embedding_model
+        setting_id=gitlab_integration.id, embeddings_model=embedding_model
     )
 
     assistant = assistant(

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/assistant/tools/git/test_assistant_with_git_tools.py

@@ -31,7 +31,7 @@ def test_assistant_with_list_branch_set_active_branch_tools(
     code_datasource,
     code_context,
     git_utils,
-    git_integration,
+    gitlab_integration,
     similarity_check,
     toolkit,
     tool_name,
@@ -42,7 +42,7 @@ def test_assistant_with_list_branch_set_active_branch_tools(
         toolkit,
         tool_name,
         context=code_context(code_datasource),
-        settings=git_integration,
+        settings=gitlab_integration,
     )
     response, triggered_tools = assistant_utils.ask_assistant(
         assistant, prompt, minimal_response=False
@@ -65,7 +65,7 @@ def test_assistant_with_create_branch_tool(
     code_datasource,
     code_context,
     git_utils,
-    git_integration,
+    gitlab_integration,
     similarity_check,
     toolkit,
     tool_name,
@@ -81,7 +81,7 @@ def test_assistant_with_create_branch_tool(
             toolkit,
             tool_name,
             context=code_context(code_datasource),
-            settings=git_integration,
+            settings=gitlab_integration,
         )
         response, triggered_tools = assistant_utils.ask_assistant(
             assistant, prompt, minimal_response=False
@@ -114,7 +114,7 @@ def test_assistant_with_create_file_tool(
     code_context,
     code_datasource,
     git_utils,
-    git_integration,
+    gitlab_integration,
     similarity_check,
     toolkit,
     tool_name,
@@ -134,7 +134,7 @@ def test_assistant_with_create_file_tool(
             toolkit,
             tool_name,
             context=code_context(code_datasource),
-            settings=git_integration,
+            settings=gitlab_integration,
         )
         response, triggered_tools = assistant_utils.ask_assistant(
             assistant, prompt, minimal_response=False
@@ -165,7 +165,7 @@ def test_assistant_with_create_merge_request_tool(
     code_context,
     code_datasource,
     git_utils,
-    git_integration,
+    gitlab_integration,
     similarity_check,
     toolkit,
     tool_name,
@@ -183,7 +183,7 @@ def test_assistant_with_create_merge_request_tool(
             toolkit,
             tool_name,
             context=code_context(code_datasource),
-            settings=git_integration,
+            settings=gitlab_integration,
         )
         response, triggered_tools = assistant_utils.ask_assistant(
             assistant, prompt, minimal_response=False
@@ -211,7 +211,7 @@ def test_assistant_with_delete_file_tool(
     code_context,
     code_datasource,
     git_utils,
-    git_integration,
+    gitlab_integration,
     datasource_utils,
     assistant_utils,
     similarity_check,
@@ -245,7 +245,7 @@ def test_assistant_with_delete_file_tool(
             toolkit,
             tool_name,
             context=code_context(code_datasource),
-            settings=git_integration,
+            settings=gitlab_integration,
         )
         response, triggered_tools = assistant_utils.ask_assistant(
             assistant, prompt, minimal_response=False
@@ -278,7 +278,7 @@ def test_assistant_with_get_merge_request_changes_tool(
     code_context,
     code_datasource,
     git_utils,
-    git_integration,
+    gitlab_integration,
     similarity_check,
     toolkit,
     tool_name,
@@ -296,7 +296,7 @@ def test_assistant_with_get_merge_request_changes_tool(
             toolkit,
             tool_name,
             context=code_context(code_datasource),
-            settings=git_integration,
+            settings=gitlab_integration,
         )
 
         create_mr_prompt = create_mr_prompt_template(source_branch, class_name, mr_name)
@@ -341,7 +341,7 @@ def test_assistant_with_update_file_tools(
     code_context,
     code_datasource,
     git_utils,
-    git_integration,
+    gitlab_integration,
     datasource_utils,
     similarity_check,
     toolkit,
@@ -361,7 +361,7 @@ def test_assistant_with_update_file_tools(
             toolkit,
             tool_name,
             context=code_context(code_datasource),
-            settings=git_integration,
+            settings=gitlab_integration,
         )
 
         create_prompt = create_prompt_template(class_name)

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/conftest.py

@@ -57,6 +57,7 @@ from codemie_test_harness.tests.utils.env_resolver import (
     EnvironmentResolver,
     get_environment,
 )
+from codemie_test_harness.tests.utils.webhook_utils import WebhookUtils
 
 logger = setup_logger(__name__)
 
@@ -94,10 +95,17 @@ def pytest_configure(config):
     This hook loads environment variables from .env file and optionally
     retrieves additional configuration from AWS Parameter Store.
     """
+    import platform
     from codemie_test_harness.cli.runner import resolve_tests_path_and_root
     from codemie_test_harness.tests.utils.aws_parameters_store import AwsParameterStore
     from codemie_test_harness.tests.utils.env_utils import EnvManager
 
+    # Set timeout method based on OS
+    # Windows requires 'thread' method, Unix-like systems can use 'signal'
+    config.option.timeout_method = (
+        "thread" if platform.system() == "Windows" else "signal"
+    )
+
     # Resolve the root directory and .env file path
     _, root_dir = resolve_tests_path_and_root()
     env_file_path = Path(root_dir) / ".env"
@@ -255,21 +263,15 @@ def conversation_utils(client):
 
 
 @pytest.fixture(scope="session")
-def git_integration(integration_utils):
-    integration = integration_utils.create_integration(
-        credential_type=CredentialTypes.GIT,
-        credential_values=CredentialsManager.gitlab_credentials(),
-    )
-    yield integration
-    if integration:
-        integration_utils.delete_integration(integration)
+def webhook_utils(client):
+    return WebhookUtils(client)
 
 
 @pytest.fixture(scope="session")
-def github_integration(integration_utils):
+def gitlab_integration(integration_utils):
     integration = integration_utils.create_integration(
         credential_type=CredentialTypes.GIT,
-        credential_values=CredentialsManager.github_credentials(),
+        credential_values=CredentialsManager.gitlab_credentials(),
     )
     yield integration
     if integration:
@@ -277,10 +279,10 @@ def github_integration(integration_utils):
 
 
 @pytest.fixture(scope="session")
-def gitlab_integration(integration_utils):
+def github_integration(integration_utils):
     integration = integration_utils.create_integration(
         credential_type=CredentialTypes.GIT,
-        credential_values=CredentialsManager.gitlab_credentials(),
+        credential_values=CredentialsManager.github_credentials(),
     )
     yield integration
     if integration:
@@ -409,6 +411,33 @@ def report_portal_integration(integration_utils):
         integration_utils.delete_integration(integration)
 
 
+@pytest.fixture(scope="module")
+def webhook_integration(integration_utils):
+    created_integrations = []
+
+    def _create(webhook_id, resource_type, resource_id, is_enabled: bool = True):
+        credential_values = [
+            CredentialValues(key="url", value=CredentialsManager.AUTO_GENERATED),
+            CredentialValues(key="webhook_id", value=webhook_id),
+            CredentialValues(key="is_enabled", value=is_enabled),
+            CredentialValues(key="resource_type", value=resource_type),
+            CredentialValues(key="resource_id", value=resource_id),
+        ]
+        integration = integration_utils.create_integration(
+            CredentialTypes.WEBHOOK, credential_values
+        )
+        created_integrations.append(integration)
+        return integration
+
+    yield _create
+
+    for integration in created_integrations:
+        try:
+            integration_utils.delete_integration(integration)
+        except HTTPError:
+            pass
+
+
 @pytest.fixture(scope="function")
 def general_integration(integration_utils):
     created_integration: Optional[Integration] = None

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/e2e/test_e2e.py

@@ -65,14 +65,14 @@ def test_assistant_with_code_kb(
     reason="Test is skipped when GIT_ENV is set to github",
 )
 def test_assistant_with_vcs_gitlab_tool(
-    assistant_utils, gitlab_integration, default_llm, similarity_check
+    assistant_utils, git_integration, default_llm, similarity_check
 ):
     assistant = assistant_utils.create_assistant(
         llm_model_type=default_llm.base_name,
         toolkits=[
             ToolKitDetails(
                 toolkit=Toolkit.VCS,
-                tools=[ToolDetails(name=VcsTool.GITLAB, settings=gitlab_integration)],
+                tools=[ToolDetails(name=VcsTool.GITLAB, settings=git_integration)],
             )
         ],
     )

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/ui/datasource/test_create_datasource.py

@@ -79,7 +79,7 @@ def test_datasource_creation_validation(page):
 
 @pytest.mark.datasource
 @pytest.mark.ui
-def test_create_git_datasource(page, git_integration):
+def test_create_git_datasource(page, gitlab_integration):
     """Test creating a new datasource with all required fields."""
     datasource_page = DataSourcePage(page)
     create_page = CreateEditDatasourcePage(page)
@@ -89,7 +89,7 @@ def test_create_git_datasource(page, git_integration):
         project_name=PROJECT,
         repo_link=CredentialsManager.get_parameter("GITLAB_PROJECT"),
         branch="main",
-        integration=git_integration.alias,
+        integration=gitlab_integration.alias,
     )
 
     datasource_page.should_see_table_row_with_values(

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/ui/datasource/test_edit_datasource.py

@@ -22,14 +22,14 @@ from codemie_test_harness.tests.utils.constants import FILES_PATH
 @pytest.mark.datasource
 @pytest.mark.ui
 def test_edit_git_datasource(
-    page, datasource_utils, git_integration, default_embedding_llm
+    page, datasource_utils, gitlab_integration, default_embedding_llm
 ):
     """Test that all main Edit Datasource page elements are visible."""
     datasource_page = DataSourcePage(page)
     edit_page = CreateEditDatasourcePage(page)
 
     datasource = datasource_utils.create_gitlab_datasource(
-        setting_id=git_integration.id,
+        setting_id=gitlab_integration.id,
         embeddings_model=default_embedding_llm.base_name,
     )
 

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/ui/datasource/test_view_datasource.py

@@ -23,13 +23,13 @@ from codemie_test_harness.tests.utils.constants import FILES_PATH
 @pytest.mark.datasource
 @pytest.mark.ui
 def test_view_git_datasource_page(
-    page, datasource_utils, git_integration, default_embedding_llm, client
+    page, datasource_utils, gitlab_integration, default_embedding_llm, client
 ):
     """Test that all main View Datasource page elements are visible."""
     datasource_page = DataSourcePage(page)
     view_page = ViewDatasourcePage(page)
     datasource = datasource_utils.create_gitlab_datasource(
-        setting_id=git_integration.id,
+        setting_id=gitlab_integration.id,
         embeddings_model=default_embedding_llm.base_name,
     )
 

{codemie_test_harness-0.1.204 → codemie_test_harness-0.1.206}/codemie_test_harness/tests/utils/constants.py

@@ -21,7 +21,7 @@ WORK_ITEM_ID_PATTERN = r"workItems/(\d+)"
 
 vcs_integrations = {
     VcsTool.GITHUB: "github_integration",
-    VcsTool.GITLAB: "gitlab_integration",
+    VcsTool.GITLAB: "git_integration",
 }
 
 project_management_integrations = {

codemie_test_harness-0.1.206/codemie_test_harness/tests/utils/webhook_utils.py

@@ -0,0 +1,10 @@
+from typing import Any, Dict
+
+from codemie_test_harness.tests.utils.base_utils import BaseUtils
+
+
+class WebhookUtils(BaseUtils):
+    """Utility class for webhook operations in CodeMie test harness."""
+
+    def trigger_webhook(self, webhook_id: str, data: Dict[str, Any] = None):
+        return self.client.webhook.trigger(webhook_id, data)

codemie_test_harness-0.1.206/codemie_test_harness/tests/webhook/test_webhook_service.py

@@ -0,0 +1,225 @@
+"""Webhook integration tests for CodeMie.
+
+This module contains tests for webhook functionality including:
+- Basic webhook operations with different resource types (assistant, workflow, datasource)
+- Error handling and validation
+- Webhook triggering verification
+"""
+
+import pytest
+
+from codemie_sdk.models.workflow import ExecutionStatus
+from hamcrest import (
+    assert_that,
+    equal_to,
+    greater_than,
+    has_length,
+    has_item,
+)
+from codemie_test_harness.tests.utils.base_utils import get_random_name, wait_for_entity
+from codemie_test_harness.tests.ui.test_data.datasource_test_data import (
+    DataSourceStatus,
+)
+
+
+@pytest.mark.webhook
+@pytest.mark.api
+class TestWebhookBasicOperations:
+    """Basic webhook operations tests - verify webhook creation and triggering for different resource types."""
+
+    def test_webhook_with_assistant(
+        self, assistant, webhook_integration, webhook_utils, conversation_utils
+    ):
+        """Test webhook can be created with assistant and triggers conversation creation."""
+        webhook_id = get_random_name()
+        message = f"Test message for {webhook_id}"
+
+        # Create assistant
+        assistant = assistant()
+
+        # Create webhook integration
+        webhook_integration(webhook_id, "assistant", assistant.id)
+
+        # Trigger webhook
+        response = webhook_utils.trigger_webhook(webhook_id, data=message)
+        assert_that(response.status_code, equal_to(200))
+        assert_that(
+            response.json()["message"], equal_to("Webhook invoked successfully")
+        )
+
+        # Wait for conversation
+        conversation = wait_for_entity(
+            lambda: conversation_utils.list_conversations(),
+            entity_name=f'"{message}"',
+        )
+
+        # Verify conversation details
+        assert_that(conversation.name.strip('"'), equal_to(message))
+        assert_that(conversation.initial_assistant_id, equal_to(assistant.id))
+        assert_that(conversation.assistant_ids, has_item(assistant.id))
+
+    def test_webhook_with_workflow(
+        self,
+        workflow_with_virtual_assistant,
+        default_llm,
+        webhook_integration,
+        webhook_utils,
+        workflow_utils,
+    ):
+        """Test webhook can be created with workflow and triggers workflow execution."""
+        webhook_id = get_random_name()
+        message = f"Test message for {webhook_id}"
+
+        # Create a simple workflow
+        workflow = workflow_with_virtual_assistant(webhook_id)
+
+        # Create webhook integration
+        webhook_integration(webhook_id, "workflow", workflow.id)
+
+        # Trigger webhook
+        response = webhook_utils.trigger_webhook(webhook_id, data=message)
+        assert_that(response.status_code, equal_to(200))
+        assert_that(
+            response.json()["message"], equal_to("Webhook invoked successfully")
+        )
+
+        # Verify workflow execution was created
+        executions = workflow_utils.get_executions(workflow)
+        assert_that(
+            executions,
+            has_length(greater_than(0)),
+            "Workflow execution should be created",
+        )
+
+        # Verify execution details
+        execution = executions[0]
+
+        assert_that(execution.prompt.strip('""'), equal_to(message))
+        assert_that(execution.workflow_id, equal_to(workflow.id))
+        assert_that(execution.status, equal_to(ExecutionStatus.IN_PROGRESS))
+
+    @pytest.mark.parametrize(
+        "datasource_fixture",
+        [
+            "jira_datasource",
+            "confluence_datasource",
+            "code_datasource",
+        ],
+    )
+    def test_webhook_with_datasource(
+        self,
+        request,
+        datasource_fixture,
+        webhook_integration,
+        webhook_utils,
+        datasource_utils,
+    ):
+        """Test webhook can be created with datasource and triggers successfully.
+
+        This test verifies webhook works with datasource as the resource type.
+        After triggering the webhook, it verifies that datasource indexing is in progress.
+
+        Test is parametrized to work with different datasource types:
+        - jira_datasource
+        - confluence_datasource
+        - code_datasource
+        """
+
+        # Get the datasource from the fixture
+        datasource = request.getfixturevalue(datasource_fixture)
+
+        # Create webhook integration with datasource
+        webhook_id = get_random_name()
+        webhook_integration(webhook_id, "datasource", datasource.id)
+
+        # Trigger webhook
+        response = webhook_utils.trigger_webhook(webhook_id)
+        assert_that(response.status_code, equal_to(200))
+        assert_that(
+            response.json()["message"], equal_to("Webhook invoked successfully")
+        )
+
+        # Verify datasource exists and check its status (indexing should be in progress)
+        triggered_datasource = datasource_utils.get_datasource(datasource.id)
+
+        assert_that(triggered_datasource.id, equal_to(datasource.id))
+        assert_that(triggered_datasource.name, equal_to(datasource.name))
+        assert_that(triggered_datasource.status, DataSourceStatus.FETCHING)
+
+
+@pytest.mark.webhook
+@pytest.mark.api
+class TestWebhookErrorHandling:
+    """Tests for webhook error handling - invalid IDs and disabled webhooks."""
+
+    def test_webhook_trigger_with_invalid_id(self, webhook_utils):
+        """Test that triggering webhook with invalid ID returns error."""
+        invalid_webhook_id = "non_existent_webhook_" + get_random_name()
+
+        response = webhook_utils.trigger_webhook(invalid_webhook_id)
+
+        # Should return error status code
+        assert_that(response.status_code, equal_to(500))
+        assert_that(response.json()["detail"], equal_to("Webhook processing failed"))
+
+    def test_webhook_trigger_with_disabled_webhook(
+        self, assistant, webhook_integration, webhook_utils, conversation_utils
+    ):
+        """Test that disabled webhook cannot be triggered."""
+        assistant = assistant()
+        webhook_id = get_random_name()
+        message = f"Test message for {webhook_id}"
+
+        # Create webhook in disabled state
+        webhook_integration(webhook_id, "assistant", assistant.id, is_enabled=False)
+
+        # Try to trigger disabled webhook
+        response = webhook_utils.trigger_webhook(webhook_id, data=message)
+
+        # Should return error status code (not 200) for disabled webhook
+        assert_that(response.status_code, equal_to(500))
+        assert_that(response.json()["detail"], equal_to("Webhook processing failed"))
+
+        conversations = conversation_utils.get_conversation_by_assistant_id(
+            assistant.id
+        )
+        assert_that(len(conversations), equal_to(0))
+
+    def test_webhook_invalid_resource_id(self, webhook_integration, webhook_utils):
+        """Test webhook with invalid resource ID."""
+        webhook_id = get_random_name()
+        message = f"Test message for {webhook_id}"
+        invalid_resource_id = "invalid_resource_" + get_random_name()
+
+        # Create webhook with invalid resource ID
+        webhook_integration(webhook_id, "assistant", invalid_resource_id)
+
+        # Trigger webhook - should handle gracefully
+        response = webhook_utils.trigger_webhook(webhook_id, data=message)
+
+        assert_that(response.status_code, equal_to(500))
+        assert_that(response.json()["detail"], equal_to("Webhook processing failed"))
+
+    def test_webhook_with_file_datasource(
+        self,
+        request,
+        file_datasource,
+        webhook_integration,
+        webhook_utils,
+        datasource_utils,
+    ):
+        """Test webhook with unsupported datasource type."""
+
+        # Create webhook integration with datasource
+        webhook_id = get_random_name()
+        webhook_integration(webhook_id, "datasource", file_datasource.id)
+
+        # Trigger webhook
+        response = webhook_utils.trigger_webhook(webhook_id)
+        assert_that(response.status_code, equal_to(403))
+        assert_that(
+            response.json()["detail"],
+            equal_to(
+                "Datasource type 'knowledge_base_file' is not supported via webhook."
+            ),
+        )