codemie-test-harness 0.1.136__py3-none-any.whl → 0.1.138__py3-none-any.whl

codemie_test_harness/tests/service/test_integration_service.py

@@ -1,98 +1,114 @@
-"""Integration tests for IntegrationService."""
-
 from time import sleep
-
 import pytest
+from hamcrest import (
+    assert_that,
+    all_of,
+    instance_of,
+    is_not,
+    equal_to,
+    less_than_or_equal_to,
+    has_length,
+    empty,
+    any_of,
+    has_property,
+    greater_than,
+)
 
-from codemie_sdk import CodeMieClient
 from codemie_sdk.models.integration import (
     Integration,
     CredentialTypes,
     CredentialValues,
     IntegrationType,
 )
-from codemie_test_harness.tests import PROJECT
+from codemie_test_harness.tests import PROJECT, GITHUB_URL
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 
 
-def test_list_project_integrations_minimal(client: CodeMieClient):
-    """Test listing project integrations with minimal response."""
-    # Get list of project integrations
-    integrations = client.integrations.list(setting_type=IntegrationType.PROJECT)
+def test_list_project_integrations_minimal_response(search_utils):
+    integrations = search_utils.list_integrations(setting_type=IntegrationType.PROJECT)
+    assert_that(integrations, all_of(instance_of(list), has_length(greater_than(0))))
 
-    # Verify we got a list of integrations
-    assert isinstance(integrations, list)
-
-    # Verify each integration has the correct type and required fields
     for integration in integrations:
-        assert isinstance(integration, Integration)
-        assert integration.project_name is not None
-        assert integration.credential_type is not None
-        assert isinstance(integration.credential_values, list)
-        assert integration.setting_type == IntegrationType.PROJECT
-
+        assert_that(
+            integration,
+            all_of(
+                instance_of(Integration),
+                has_property("project_name", has_length(greater_than(0))),
+                has_property("credential_type", is_not(any_of(None, empty()))),
+                has_property("credential_values", has_length(greater_than(0))),
+                has_property("setting_type", equal_to(IntegrationType.PROJECT)),
+            ),
+        )
 
-def test_list_user_integrations_minimal(client: CodeMieClient):
-    """Test listing user integrations with minimal response."""
-    # Get list of user integrations
-    integrations = client.integrations.list(setting_type=IntegrationType.USER)
 
-    # Verify we got a list of integrations
-    assert isinstance(integrations, list)
+def test_list_user_integrations_minimal_response(search_utils):
+    integrations = search_utils.list_integrations()
+    assert_that(integrations, all_of(instance_of(list), has_length(greater_than(0))))
 
-    # Verify each integration has the correct type and required fields
     for integration in integrations:
-        assert isinstance(integration, Integration)
-        assert integration.project_name is not None
-        assert integration.credential_type is not None
-        assert isinstance(integration.credential_values, list)
-        assert integration.setting_type == IntegrationType.USER
+        assert_that(
+            integration,
+            all_of(
+                instance_of(Integration),
+                has_property("project_name", is_not(None)),
+                has_property("credential_type", is_not(any_of(None, empty()))),
+                has_property("credential_values", has_length(greater_than(0))),
+                has_property("setting_type", equal_to(IntegrationType.USER)),
+            ),
+        )
 
 
-def test_list_integrations_with_filters(client: CodeMieClient):
-    """Test listing integrations with filters for both types."""
-    # Test project integrations with filters
+def test_list_integrations_with_filters(search_utils):
     filters = {"type": CredentialTypes.GIT}
-    project_integrations = client.integrations.list(
+
+    project_integrations = search_utils.list_integrations(
         setting_type=IntegrationType.PROJECT, filters=filters
     )
-
     for integration in project_integrations:
-        assert integration.credential_type == CredentialTypes.GIT
-        assert integration.setting_type == IntegrationType.PROJECT
+        assert_that(
+            integration,
+            all_of(
+                has_property("credential_type", equal_to(CredentialTypes.GIT)),
+                has_property("setting_type", equal_to(IntegrationType.PROJECT)),
+            ),
+        )
 
-    # Test user integrations with filters
-    user_integrations = client.integrations.list(
+    user_integrations = search_utils.list_integrations(
         setting_type=IntegrationType.USER, filters=filters
     )
-
     for integration in user_integrations:
-        assert integration.credential_type == CredentialTypes.GIT
-        assert integration.setting_type == IntegrationType.USER
+        assert_that(
+            integration,
+            all_of(
+                has_property("credential_type", equal_to(CredentialTypes.GIT)),
+                has_property("setting_type", equal_to(IntegrationType.USER)),
+            ),
+        )
 
 
-def test_list_integrations_pagination(client: CodeMieClient):
-    """Test integration listing with pagination for both types."""
+def test_list_integrations_with_pagination(search_utils):
     for setting_type in [IntegrationType.PROJECT, IntegrationType.USER]:
         # Get first page with 5 items
-        page_1 = client.integrations.list(setting_type=setting_type, page=0, per_page=5)
-        assert len(page_1) <= 5
+        page_1 = search_utils.list_integrations(
+            setting_type=setting_type, page=0, per_page=5
+        )
+        assert_that(len(page_1), less_than_or_equal_to(5))
 
         # Get second page with 5 items
-        page_2 = client.integrations.list(setting_type=setting_type, page=1, per_page=5)
-        assert len(page_2) <= 5
+        page_2 = search_utils.list_integrations(
+            setting_type=setting_type, page=1, per_page=5
+        )
+        assert_that(len(page_2), less_than_or_equal_to(5))
 
         # Verify pages contain different integrations
         if page_1 and page_2:
-            assert page_1[0].id != page_2[0].id
+            assert_that(page_1[0].id, is_not(equal_to(page_2[0].id)))
 
 
 @pytest.mark.parametrize(
     "setting_type", [IntegrationType.PROJECT, IntegrationType.USER]
 )
-def test_integration_lifecycle(client: CodeMieClient, setting_type: IntegrationType):
-    """Test full integration lifecycle for both user and project settings."""
-    # Step 1: Create test integration
+def test_integration_full_lifecycle(integration_utils, setting_type: IntegrationType):
     test_project = PROJECT
     test_alias = get_random_name()
 
@@ -108,76 +124,73 @@ def test_integration_lifecycle(client: CodeMieClient, setting_type: IntegrationT
         setting_type=setting_type,
     )
 
-    # Create integration
-    created = client.integrations.create(create_request)
-    assert created is not None
-    sleep(5)
+    created = integration_utils.send_create_integration_request(create_request)
 
-    # Step 2: Verify integration exists in the list
-    found = client.integrations.get_by_alias(test_alias, setting_type=setting_type)
-    assert found is not None
-    assert found.alias == test_alias
-    assert found.setting_type == setting_type
-    assert found.credential_values[0].value == "https://github.com"
-    assert found.credential_values[1].value == "test-token-name"
-
-    try:
-        # Step 3: Update the integration
-        updated_alias = f"{test_alias} Updated"
-        update_request = Integration(
-            project_name=test_project,
-            credential_type=CredentialTypes.GIT,
-            credential_values=[
-                CredentialValues(
-                    key="url", value="https://github.com/test/repo-updated"
-                ),
-                CredentialValues(key="token_name", value="test-token-name-updated"),
-            ],
-            alias=updated_alias,
-            setting_type=setting_type,
-        )
+    assert_that(created, is_not(None))
+    sleep(5)  # Verify integration creation
 
-        updated = client.integrations.update(found.id, update_request)
-        assert updated is not None
-        sleep(5)
+    found_integration = integration_utils.get_integration_by_alias(
+        integration_alias=test_alias, integration_type=setting_type
+    )
 
-        # Step 4: Get and verify updated integration
-        updated_integration = client.integrations.get_by_alias(
-            updated_alias, setting_type=setting_type
-        )
-        assert updated_integration is not None
-        assert updated_integration.id == found.id
-        assert updated_integration.alias == updated_alias
-        assert updated_integration.setting_type == setting_type
-        assert updated_integration.credential_values[0].value == "https://github.com"
-        assert (
-            updated_integration.credential_values[1].value == "test-token-name-updated"
-        )
+    assert_that(
+        found_integration,
+        all_of(
+            has_property("alias", equal_to(test_alias)),
+            has_property("setting_type", equal_to(setting_type)),
+        ),
+    )
+    assert_that(found_integration.credential_values[0].value, equal_to(GITHUB_URL))
+    assert_that(
+        found_integration.credential_values[1].value, equal_to("test-token-name")
+    )
+
+    # Updating the integration
+    updated_alias = f"{test_alias} Updated"
+    update_request = Integration(
+        id=found_integration.id,
+        project_name=test_project,
+        credential_type=CredentialTypes.GIT,
+        credential_values=[
+            CredentialValues(key="url", value="https://github.com/test/repo-updated"),
+            CredentialValues(key="token_name", value="test-token-name-updated"),
+        ],
+        alias=updated_alias,
+        setting_type=setting_type,
+    )
+
+    updated = integration_utils.update_integration(update_request)
+
+    assert_that(updated, is_not(None))
+    sleep(5)
 
-    finally:
-        # Step 5: Clean up - delete created integration
-        if found:
-            try:
-                client.integrations.delete(found.id, setting_type=setting_type)
-                sleep(5)
-                # Verify deletion
-                with pytest.raises(Exception) as exc_info:
-                    client.integrations.get(found.id, setting_type=setting_type)
-                assert (
-                    "404" in str(exc_info.value)
-                    or "not found" in str(exc_info.value).lower()
-                )
-            except Exception as e:
-                pytest.fail(f"Failed to clean up integration: {str(e)}")
+    updated_integration = integration_utils.get_integration_by_alias(
+        integration_alias=updated_alias, integration_type=setting_type
+    )
+    assert_that(
+        updated_integration,
+        all_of(
+            has_property("id", equal_to(found_integration.id)),
+            has_property("alias", equal_to(updated_alias)),
+            has_property("setting_type", equal_to(setting_type)),
+        ),
+    )
+    assert_that(
+        updated_integration.credential_values[0].value,
+        equal_to(GITHUB_URL),
+    )
+    assert_that(
+        updated_integration.credential_values[1].value,
+        equal_to("test-token-name-updated"),
+    )
 
 
 @pytest.mark.parametrize(
     "setting_type", [IntegrationType.PROJECT, IntegrationType.USER]
 )
-def test_create_integration_invalid_data(
-    client: CodeMieClient, setting_type: IntegrationType
+def test_create_integration_with_invalid_data(
+    integration_utils, setting_type: IntegrationType
 ):
-    """Test creating integration with invalid data for both types."""
     with pytest.raises(Exception):
         invalid_request = Integration(
             project_name="",  # Invalid - empty project name
@@ -185,26 +198,24 @@ def test_create_integration_invalid_data(
             credential_values=[],  # Invalid - empty credentials
             setting_type=setting_type,
         )
-        client.integrations.create(invalid_request)
+        integration_utils.send_create_integration_request(invalid_request)
 
 
 @pytest.mark.parametrize(
     "setting_type", [IntegrationType.PROJECT, IntegrationType.USER]
 )
-def test_update_integration_invalid_data(
-    client: CodeMieClient, setting_type: IntegrationType
+def test_update_integration_with_invalid_data(
+    search_utils, integration_utils, setting_type: IntegrationType
 ):
-    """Test updating integration with invalid data for both types."""
-    # First, get a valid integration ID
-    integrations = client.integrations.list(setting_type=setting_type)
-    assert len(integrations) > 0
-    test_id = integrations[0].id
+    integrations = search_utils.list_integrations(setting_type=setting_type)
+    assert_that(integrations, has_length(greater_than(0)))
 
     with pytest.raises(Exception):
         invalid_request = Integration(
+            id=integrations[0].id,
             project_name="",  # Invalid - empty project name
             credential_type=CredentialTypes.GIT,
             credential_values=[],  # Invalid - empty credentials
             setting_type=setting_type,
         )
-        client.integrations.update(test_id, invalid_request)
+        integration_utils.update_integration(invalid_request)

codemie_test_harness/tests/service/test_llm_service.py

@@ -1,20 +1,19 @@
-"""Integration tests for LLMService."""
-
-from codemie_sdk import CodeMieClient
+import pytest
 from codemie_sdk.models.llm import LLMModel
+from hamcrest import (
+    assert_that,
+    instance_of,
+    has_length,
+    greater_than,
+    all_of,
+)
 
 
-def test_list_llm_models(client: CodeMieClient):
-    """Test successful retrieval of available LLM models."""
-    models = client.llms.list()
-    assert isinstance(models, list)
-    assert len(models) > 0
-    assert all(isinstance(model, LLMModel) for model in models)
-
-
-def test_list_embeddings_models(client: CodeMieClient):
-    """Test successful retrieval of available embeddings models."""
-    models = client.llms.list_embeddings()
-    assert isinstance(models, list)
-    assert len(models) > 0
-    assert all(isinstance(model, LLMModel) for model in models)
+@pytest.mark.parametrize(
+    "model_list_function", ["list_llm_models", "list_embedding_llm_models"]
+)
+def test_list_available_models(llm_utils, model_list_function):
+    models = getattr(llm_utils, model_list_function)()
+    assert_that(models, all_of(instance_of(list), has_length(greater_than(0))))
+    for model in models:
+        assert_that(model, instance_of(LLMModel))

codemie_test_harness/tests/service/test_task_service.py

@@ -1,14 +1,19 @@
-"""Integration tests for TaskService."""
-
 import time
 import uuid
 from datetime import datetime
 
 import pytest
+from hamcrest import (
+    assert_that,
+    has_length,
+    greater_than,
+    equal_to,
+    is_in,
+    contains_string,
+    is_not,
+)
 
-from codemie_sdk import CodeMieClient
 from codemie_sdk.models.assistant import (
-    AssistantCreateRequest,
     AssistantChatRequest,
     ToolKitDetails,
     ToolDetails,
@@ -16,136 +21,119 @@ from codemie_sdk.models.assistant import (
     ChatRole,
 )
 from codemie_sdk.models.task import BackgroundTaskEntity
-from codemie_test_harness.tests import PROJECT, LANGFUSE_TRACES_ENABLED
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests import PROJECT
 
 
-def test_background_task_flow(client: CodeMieClient, default_llm):
-    """Test the complete flow of a background task with assistant chat."""
-    # Step 1: Get available toolkits and tools
-    toolkits = client.assistants.get_tools()
-    assert len(toolkits) > 0, "At least one toolkit is required for testing"
+def test_run_flow_of_background_task(assistant_utils, default_llm):
+    toolkits = assistant_utils.get_assistant_tools()
+    assert_that(
+        toolkits,
+        has_length(greater_than(0)),
+        "At least one toolkit is required for testing",
+    )
 
     first_toolkit = toolkits[0]
-    assert len(first_toolkit.tools) > 0, "No tools in the first toolkit"
+    assert_that(
+        first_toolkit.tools,
+        has_length(greater_than(0)),
+        "No tools in the first toolkit",
+    )
     first_tool = first_toolkit.tools[0]
 
-    # Step 2: Create assistant for testing
-    assistant_project = PROJECT
-    assistant_name = get_random_name()
-    request = AssistantCreateRequest(
-        name=assistant_name,
-        slug=assistant_name,
-        description="Integration test assistant for background tasks",
-        system_prompt="You are a helpful integration test assistant. Please provide detailed responses.",
+    created = assistant_utils.create_assistant(
         llm_model_type=default_llm.base_name,
-        project=assistant_project,
         toolkits=[
             ToolKitDetails(
                 toolkit=first_toolkit.toolkit, tools=[ToolDetails(name=first_tool.name)]
             )
         ],
+        system_prompt="You are a helpful integration test assistant. Please provide detailed responses.",
+        description="Integration test assistant for background tasks",
     )
+    filters = {"project": PROJECT, "shared": False}
+    assistants = assistant_utils.get_assistants(minimal_response=True, filters=filters)
+    found_assistant = next((a for a in assistants if a.name == created.name), None)
+
+    # Start a chat in background mode with a complex question
+    complex_question = """
+    Please provide a detailed analysis of software architecture patterns, including:
+    1. Monolithic Architecture
+    2. Microservices Architecture
+    3. Event-Driven Architecture
+    4. Layered Architecture
+    5. Space-Based Architecture
+    
+    For each pattern, include:
+    - Definition
+    - Key characteristics
+    - Advantages and disadvantages
+    - Best use cases
+    - Implementation challenges
+    - Real-world examples
+    """
+
+    chat_request = AssistantChatRequest(
+        text=complex_question,
+        conversation_id=str(uuid.uuid4()),
+        history=[
+            ChatMessage(
+                role=ChatRole.USER,
+                message="Hi, I need help with software architecture",
+            ),
+            ChatMessage(
+                role=ChatRole.ASSISTANT,
+                message="Of course! I'd be happy to help with software architecture. What would you like to know?",
+            ),
+        ],
+        stream=False,
+        background_task=True,  # Enable background mode
+    )
+
+    response = assistant_utils.send_chat_request(
+        assistant=found_assistant, request=chat_request
+    )
+
+    assert_that(response.task_id, is_not(None))
+
+    # Poll task status until completion
+    max_attempts = 30  # Maximum number of polling attempts
+    polling_interval = 2  # Seconds between polling attempts
+    task_id = response.task_id
+    task_completed = False
+
+    for _ in range(max_attempts):
+        task = assistant_utils.get_tasks(task_id)
+        assert_that(isinstance(task, BackgroundTaskEntity))
+        assert_that(task.id, equal_to(task_id))
+        assert_that(isinstance(task.date, datetime))
+        assert_that(isinstance(task.update_date, datetime))
+        assert_that(task.status, is_in(["STARTED", "COMPLETED", "FAILED"]))
+        assert_that(task.user, is_not(None))
+        assert_that(task.task, is_not(None))
+
+        if task.status == "COMPLETED":
+            task_completed = True
+            assert_that(len(task.final_output), greater_than(0))
+
+            # The response should contain architecture patterns
+            assert_that(task.final_output, contains_string("Monolithic"))
+            assert_that(task.final_output, contains_string("Microservices"))
+            break
+        elif task.status == "FAILED":
+            pytest.fail(f"Task failed with output: {task.final_output}")
+
+        time.sleep(polling_interval)
+
+    assert_that(
+        task_completed,
+        equal_to(True),
+        "Task did not complete within the expected time",
+    )
+
 
-    # Create assistant
-    client.assistants.create(request)
-    filters = {"project": assistant_project, "shared": False}
-    assistants = client.assistants.list(minimal_response=True, filters=filters)
-    found_assistant = next((a for a in assistants if a.name == assistant_name), None)
-
-    try:
-        # Step 3: Start a chat in background mode with a complex question
-        complex_question = """
-        Please provide a detailed analysis of software architecture patterns, including:
-        1. Monolithic Architecture
-        2. Microservices Architecture
-        3. Event-Driven Architecture
-        4. Layered Architecture
-        5. Space-Based Architecture
-        
-        For each pattern, include:
-        - Definition
-        - Key characteristics
-        - Advantages and disadvantages
-        - Best use cases
-        - Implementation challenges
-        - Real-world examples
-        """
-
-        chat_request = AssistantChatRequest(
-            text=complex_question,
-            conversation_id=str(uuid.uuid4()),
-            history=[
-                ChatMessage(
-                    role=ChatRole.USER,
-                    message="Hi, I need help with software architecture",
-                ),
-                ChatMessage(
-                    role=ChatRole.ASSISTANT,
-                    message="Of course! I'd be happy to help with software architecture. What would you like to know?",
-                ),
-            ],
-            stream=False,
-            background_task=True,  # Enable background mode
-            metadata={"langfuse_traces_enabled": LANGFUSE_TRACES_ENABLED},
-        )
-
-        # Send chat request
-        response = client.assistants.chat(
-            assistant_id=found_assistant.id, request=chat_request
-        )
-        # Verify response contains task ID
-        assert response is not None
-        assert response.task_id is not None
-
-        # Step 4: Poll task status until completion
-        max_attempts = 30  # Maximum number of polling attempts
-        polling_interval = 2  # Seconds between polling attempts
-        task_id = response.task_id
-        task_completed = False
-
-        for _ in range(max_attempts):
-            # Get task status
-            task = client.tasks.get(task_id)
-            assert isinstance(task, BackgroundTaskEntity)
-
-            # Verify task structure
-            assert task.id == task_id
-            assert isinstance(task.date, datetime)
-            assert isinstance(task.update_date, datetime)
-            assert task.status in ["STARTED", "COMPLETED", "FAILED"]
-            assert task.user is not None
-            assert task.task is not None
-
-            if task.status == "COMPLETED":
-                task_completed = True
-                # Verify task output
-                assert task.final_output is not None
-                assert len(task.final_output) > 0
-                # The response should contain architecture patterns
-                assert "Monolithic" in task.final_output
-                assert "Microservices" in task.final_output
-                break
-            elif task.status == "FAILED":
-                pytest.fail(f"Task failed with output: {task.final_output}")
-
-            time.sleep(polling_interval)
-
-        assert task_completed, "Task did not complete within the expected time"
-
-    finally:
-        # Clean up - delete created assistant
-        if found_assistant:
-            try:
-                client.assistants.delete(found_assistant.id)
-            except Exception as e:
-                pytest.fail(f"Failed to clean up assistant: {str(e)}")
-
-
-@pytest.mark.skip(reason="Need to fix API to return 404")
-def test_get_task_not_found(client: CodeMieClient):
+def test_get_task_not_found(assistant_utils):
     """Test getting a non-existent task."""
     with pytest.raises(Exception) as exc_info:
-        client.tasks.get("non-existent-task-id")
+        assistant_utils.get_tasks("non-existent-task-id")
     # assert "404" in str(exc_info.value) or "not found" in str(exc_info.value).lower(), Need to fix API to produce error
-    assert "503" in str(exc_info.value)
+    assert_that(str(exc_info.value), contains_string("500"))