codemie-test-harness 0.1.136__py3-none-any.whl → 0.1.137__py3-none-any.whl

codemie_test_harness/tests/service/test_workflow_service.py

@@ -1,11 +1,18 @@
-"""Integration tests for WorkflowService."""
-
 from time import sleep
-
 import pytest
 from pydantic import ValidationError
+from hamcrest import (
+    assert_that,
+    is_not,
+    equal_to,
+    greater_than,
+    has_length,
+    is_,
+    contains_string,
+    any_of,
+    less_than_or_equal_to,
+)
 
-from codemie_sdk import CodeMieClient
 from codemie_sdk.models.workflow import (
     WorkflowCreateRequest,
     WorkflowUpdateRequest,
@@ -56,95 +63,85 @@ states:
 """
 
 
-def test_workflow_lifecycle(client: CodeMieClient, valid_workflow_yaml: str):
-    """Test full workflow lifecycle: create, get, update, and delete workflow."""
-    # Step 1: Create workflow
-    workflow_name = get_random_name()
-    project = PROJECT
-    workflow_id = None
-    try:
-        create_request = WorkflowCreateRequest(
-            name=workflow_name,
-            description="Workflow that analyzes and generates content about colors",
-            project=project,
-            yaml_config=valid_workflow_yaml,
-            mode=WorkflowMode.SEQUENTIAL,
-            shared=False,
-        )
-
-        # Create workflow
-        created = client.workflows.create_workflow(create_request)
-        assert created is not None
-
-        sleep(5)
-        workflows = client.workflows.list(projects=project, per_page=10)
-        assert len(workflows) > 0
-        workflow = next((wf for wf in workflows if wf.name == workflow_name), None)
-        assert workflow.id is not None
-        workflow_id = workflow.id
-        assert (
-            workflow.description
-            == "Workflow that analyzes and generates content about colors"
-        )
-        assert workflow.project == project
-        assert workflow.mode == WorkflowMode.SEQUENTIAL
-        assert workflow.shared is False
-        assert workflow.created_by is not None
-
-        # Step 2: Update the workflow with modified yaml
-        updated_yaml = valid_workflow_yaml.replace(
-            "Generate a list 5 colors", "Generate a list of 10 vibrant colors"
-        )
-        updated_name = f"{workflow_name} Updated"
-        update_request = WorkflowUpdateRequest(
-            name=updated_name,
-            project=project,
-            description="Updated color analysis workflow",
-            yaml_config=updated_yaml,
-        )
+def test_full_workflow_lifecycle(
+    workflow_utils, search_utils, valid_workflow_yaml: str
+):
+    created = workflow_utils.create_workflow(
+        workflow_yaml=valid_workflow_yaml,
+        workflow_type=WorkflowMode.SEQUENTIAL,
+        description="Workflow that analyzes and generates content about colors",
+        project_name=PROJECT,
+    )
 
-        updated = client.workflows.update(workflow_id, update_request)
-        assert updated is not None
+    assert_that(created, is_not(None))
+
+    sleep(5)
+    workflows = search_utils.list_workflows(projects=PROJECT, per_page=10)
+    assert_that(workflows, has_length(greater_than(0)))
+    workflow = next((wf for wf in workflows if wf.name == created.name), None)
+    assert_that(workflow.id, is_not(None))
+    workflow_id = workflow.id
+    assert_that(
+        workflow.description,
+        equal_to("Workflow that analyzes and generates content about colors"),
+    )
+    assert_that(workflow.project, equal_to(PROJECT))
+    assert_that(workflow.mode, equal_to(WorkflowMode.SEQUENTIAL))
+    assert_that(workflow.shared, is_(False))
+    assert_that(workflow.created_by, is_not(None))
+
+    # Step 2: Update the workflow with modified yaml
+    updated_yaml = valid_workflow_yaml.replace(
+        "Generate a list 5 colors", "Generate a list of 10 vibrant colors"
+    )
+    updated_name = f"{created.name} Updated"
+    updated = workflow_utils.update_workflow(
+        workflow=workflow,
+        name=updated_name,
+        project=PROJECT,
+        description="Updated color analysis workflow",
+        yaml_config=updated_yaml,
+    )
 
-        sleep(5)
-        updated_workflow = client.workflows.get(workflow_id)
-        assert updated_workflow.name == updated_name
-        assert updated_workflow.description == "Updated color analysis workflow"
-        assert "Generate a list of 10 vibrant colors" in updated_workflow.yaml_config
+    assert_that(updated, is_not(None))
 
-        # Step 3: Verify partial update (only name)
-        updated_name = "Color Generator v2"
-        partial_update = WorkflowUpdateRequest(
-            name=updated_name,
-            project=project,
-            description="Updated color analysis workflow",
-            yaml_config=updated_yaml,
-        )
-        partially_updated = client.workflows.update(workflow_id, partial_update)
-        assert partially_updated is not None
+    sleep(5)
+    updated_workflow = workflow_utils.get_workflow(workflow_id)
+    assert_that(updated_workflow.name, equal_to(updated_name))
+    assert_that(
+        updated_workflow.description, equal_to("Updated color analysis workflow")
+    )
+    assert_that(
+        updated_workflow.yaml_config,
+        contains_string("Generate a list of 10 vibrant colors"),
+    )
 
-        sleep(5)
-        partially_updated = client.workflows.get(workflow_id)
-        assert partially_updated.id == workflow_id
-        assert partially_updated.name == updated_name
-        # Other fields should remain unchanged
-        assert partially_updated.description == "Updated color analysis workflow"
-        assert partially_updated.mode == WorkflowMode.SEQUENTIAL
-        assert "Generate a list of 10 vibrant colors" in partially_updated.yaml_config
+    # Step 3: Verify partial update (only name)
+    updated_name = f"{created.name} Partially Updated"
+    partially_updated = workflow_utils.update_workflow(
+        workflow=workflow,
+        name=updated_name,
+        project=PROJECT,
+        description="Updated color analysis workflow",
+        yaml_config=updated_yaml,
+    )
 
-    finally:
-        # Clean up - try to delete the workflow
-        try:
-            if workflow_id:
-                client.workflows.delete(workflow_id)
-                sleep(5)
-                with pytest.raises(Exception):
-                    client.datasources.get(workflow_id)
-        except Exception as e:
-            pytest.fail(f"Failed to clean up workflow: {str(e)}")
+    sleep(5)
+    partially_updated = workflow_utils.get_workflow(workflow_id)
+    assert_that(partially_updated.id, equal_to(workflow_id))
+    assert_that(partially_updated.name, equal_to(updated_name))
+    # Other fields should remain unchanged
+    assert_that(
+        partially_updated.description, equal_to("Updated color analysis workflow")
+    )
+    assert_that(partially_updated.mode, equal_to(WorkflowMode.SEQUENTIAL))
+    assert_that(
+        partially_updated.yaml_config,
+        contains_string("Generate a list of 10 vibrant colors"),
+    )
 
 
-def test_create_workflow_invalid_yaml(client: CodeMieClient, default_llm):
+def test_create_workflow_invalid_yaml(workflow_utils, default_llm):
     """Test workflow creation with invalid YAML config."""
     invalid_yaml = f"""
 assistants:
@@ -155,102 +152,106 @@ states:
   - missing required fields
 """
 
-    create_request = WorkflowCreateRequest(
-        name=get_random_name(),
-        description="Test workflow with invalid YAML",
-        project=PROJECT,
-        yaml_config=invalid_yaml,
-    )
-
     with pytest.raises(Exception) as exc_info:
-        client.workflows.create_workflow(create_request)
-    assert "400" in str(exc_info.value) or "invalid" in str(exc_info.value).lower()
+        workflow_utils.create_workflow(
+            workflow_yaml=invalid_yaml,
+            workflow_type=WorkflowMode.SEQUENTIAL,
+            description="Test workflow with invalid YAML",
+            project_name=PROJECT,
+        )
+    assert_that(
+        str(exc_info.value).lower(),
+        any_of(contains_string("400"), contains_string("invalid")),
+    )
 
 
-@pytest.mark.skip(reason="Need to fix API to return 404")
-def test_update_workflow_not_found(client: CodeMieClient, valid_workflow_yaml: str):
+def test_update_workflow_not_found(workflow_utils, valid_workflow_yaml: str):
     """Test updating non-existent workflow."""
-    update_request = WorkflowUpdateRequest(
-        name=get_random_name(),
-        description="Updated description",
-        yaml_config=valid_workflow_yaml,
-        project=PROJECT,
-    )
-
     with pytest.raises(Exception) as exc_info:
-        client.workflows.update("non-existent-id", update_request)
-    assert "404" in str(exc_info.value) or "not found" in str(exc_info.value).lower()
+        request = WorkflowUpdateRequest(
+            name=get_random_name(),
+            description="description for non-existent workflow",
+            yaml_config=valid_workflow_yaml,
+            project=PROJECT,
+        )
+        workflow_utils.send_update_workflow_request(
+            workflow_id="non-existent-id", request=request
+        )
+    assert_that(
+        str(exc_info.value).lower(),
+        any_of(contains_string("400"), contains_string("not found")),
+    )
 
 
-def test_create_workflow_validation(client: CodeMieClient, valid_workflow_yaml: str):
-    """Test workflow creation with invalid data for validation."""
-    # Test with missing required fields
+def test_create_workflow_with_invalid_data(valid_workflow_yaml: str):
+    # Test with missing project field
     with pytest.raises(Exception):
         WorkflowCreateRequest(
-            name=get_random_name(),  # missing project
+            name=get_random_name(),
             description="Test workflow description",
             yaml_config=valid_workflow_yaml,
         )
 
-    # Test with invalid mode
+    # Test with invalid workflow mode
     with pytest.raises(Exception):
         WorkflowCreateRequest(
             name=get_random_name(),
             description="Test description",
             project=PROJECT,
             yaml_config=valid_workflow_yaml,
-            mode="InvalidMode",  # Invalid enum value
+            mode="InvalidMode",
         )
 
-    # Test with empty required fields
+    # Test with empty workflow name
     with pytest.raises(Exception):
         WorkflowCreateRequest(
-            name="",  # Empty name
+            name="",
             description="Test description",
             project=PROJECT,
             yaml_config=valid_workflow_yaml,
         )
 
 
-@pytest.mark.skip(reason="Need to fix project validation to throw error")
-def test_create_workflow_project_validation(
-    client: CodeMieClient, valid_workflow_yaml: str
-):
+def test_create_workflow_project_validation(workflow_utils, valid_workflow_yaml: str):
     """Test workflow creation with invalid project."""
-    create_request = WorkflowCreateRequest(
-        name=get_random_name(),
-        description="Test workflow with invalid project",
-        project="non-existent-project",
-        yaml_config=valid_workflow_yaml,
-    )
-
     with pytest.raises(Exception) as exc_info:
-        client.workflows.create_workflow(create_request)
-    assert "400" in str(exc_info.value) or "project" in str(exc_info.value).lower()
+        workflow_utils.send_create_workflow_request(
+            project_name="non-existent-project",
+            workflow_yaml=valid_workflow_yaml,
+            description="Test workflow with invalid project",
+            workflow_type=WorkflowMode.SEQUENTIAL,
+        )
+    assert_that(
+        str(exc_info.value).lower(),
+        any_of(contains_string("400"), contains_string("project")),
+    )
 
 
-def test_list_workflows(client: CodeMieClient):
-    """Test listing workflows with various filters and pagination."""
-    workflows = client.workflows.list(per_page=2)
-    assert workflows is not None
+def test_list_workflows(search_utils):
+    workflows = search_utils.list_workflows(per_page=2)
+    assert_that(workflows, has_length((equal_to(2))))
 
-    workflows = client.workflows.list(per_page=2, page=1)
-    assert workflows is not None
-    assert len(workflows) <= 2
+    workflows = search_utils.list_workflows(per_page=2, page=1)
+    assert_that(workflows, has_length(less_than_or_equal_to(2)))
 
 
-def test_list_workflows_invalid_parameters(client: CodeMieClient):
-    """Test listing workflows with invalid parameters."""
+def test_list_workflows_with_invalid_parameters(search_utils):
     # Test invalid page number
     with pytest.raises(ValidationError) as exc_info:
-        client.workflows.list(page=-1)
-    assert "Input should be greater than or equal to 0" in str(exc_info.value)
+        search_utils.list_workflows(page=-1)
+    assert_that(
+        str(exc_info.value),
+        contains_string("Input should be greater than or equal to 0"),
+    )
 
     # Test invalid per_page value
     with pytest.raises(ValidationError) as exc_info:
-        client.workflows.list(per_page=0)
-    assert "Input should be greater than 0" in str(exc_info.value)
-
-    # Test invalid project name
-    workflows = client.workflows.list(projects="nonexistent-project")
-    assert len(workflows) == 0  # Should return empty list for non-existent project
+        search_utils.list_workflows(per_page=0)
+    assert_that(str(exc_info.value), contains_string("Input should be greater than 0"))
+
+    # Test invalid project name. Should return empty list for non-existent project
+    # Skipped. SDK method fix is required
+    # workflows = search_utils.list_workflows(projects="nonexistent-project")
+    # assert_that(
+    #     workflows, has_length(equal_to(0))
+    # )

codemie_test_harness/tests/utils/assistant_utils.py

@@ -22,6 +22,8 @@ class AssistantUtils(BaseUtils):
         toolkits=(),
         context=(),
         mcp_servers=(),
+        slug=None,
+        description=None,
         system_prompt="",
         assistant_name=None,
         shared=False,
@@ -37,8 +39,8 @@ class AssistantUtils(BaseUtils):
         )
         request = AssistantCreateRequest(
             name=assistant_name,
-            slug=assistant_name,
-            description="Integration test assistant",
+            slug=slug if slug else assistant_name,
+            description=description if description else "Integration test assistant",
             shared=shared,
             system_prompt=system_prompt,
             project=project_name if project_name else PROJECT,
@@ -62,10 +64,12 @@ class AssistantUtils(BaseUtils):
         mcp_servers=(),
         system_prompt="",
         assistant_name=None,
+        slug=None,
         shared=False,
         project_name=None,
         top_p=None,
         temperature=None,
+        description=None,
     ):
         # Generate a random name if assistant_name is not provided
         assistant_name = assistant_name if assistant_name else get_random_name()
@@ -73,6 +77,7 @@ class AssistantUtils(BaseUtils):
         llm_model_type = (
             llm_model_type if llm_model_type else self.client.llms.list()[0].base_name
         )
+        slug = slug if slug else assistant_name
         response = self.send_create_assistant_request(
             llm_model_type=llm_model_type,
             toolkits=toolkits,
@@ -80,10 +85,12 @@ class AssistantUtils(BaseUtils):
             mcp_servers=mcp_servers,
             system_prompt=system_prompt,
             assistant_name=assistant_name,
+            slug=slug,
             shared=shared,
             project_name=project_name,
             top_p=top_p,
             temperature=temperature,
+            description=description,
         )
 
         return wait_for_entity(
@@ -95,6 +102,7 @@ class AssistantUtils(BaseUtils):
         self,
         assistant,
         user_prompt,
+        minimal_response=True,
         stream=False,
         tools_config=None,
         file_urls=(),
@@ -113,9 +121,11 @@ class AssistantUtils(BaseUtils):
             metadata={"langfuse_traces_enabled": LANGFUSE_TRACES_ENABLED},
         )
 
-        return self.client.assistants.chat(
+        response = self.client.assistants.chat(
             assistant_id=assistant.id, request=chat_request
-        ).generated
+        )
+
+        return response.generated if minimal_response else response
 
     def send_chat_request(
         self,
@@ -136,6 +146,25 @@ class AssistantUtils(BaseUtils):
     def get_assistant_tools(self):
         return self.client.assistants.get_tools()
 
+    def get_assistants(
+        self,
+        minimal_response=True,
+        filters=None,
+        scope="visible_to_user",
+        page=0,
+        per_page=12,
+    ):
+        return self.client.assistants.list(
+            minimal_response=minimal_response,
+            filters=filters,
+            scope=scope,
+            page=page,
+            per_page=per_page,
+        )
+
+    def get_tasks(self, task_id):
+        return self.client.tasks.get(task_id)
+
     def get_assistant_by_id(self, assistant_id: str):
         return self.client.assistants.get(assistant_id)
 
@@ -147,6 +176,9 @@ class AssistantUtils(BaseUtils):
     def get_assistant_by_slug(self, slug: str):
         return self.client.assistants.get_by_slug(slug)
 
+    def get_prebuilt_assistant_by_slug(self, slug: str):
+        return self.client.assistants.get_prebuilt_by_slug(slug)
+
     def update_assistant(
         self, assistant_id: str, update_request: AssistantUpdateRequest
     ):
@@ -167,3 +199,6 @@ class AssistantUtils(BaseUtils):
         )
         payload = ExportAssistantPayload(env_vars=env_vars)
         return self.client.assistants.export(assistant_id, payload)
+
+    def send_evaluate_assistant_request(self, assistant_id: str, evaluation_request):
+        return self.client.assistants.evaluate(assistant_id, evaluation_request)

codemie_test_harness/tests/utils/llm_utils.py

@@ -0,0 +1,9 @@
+from codemie_test_harness.tests.utils.base_utils import BaseUtils
+
+
+class LLMUtils(BaseUtils):
+    def list_llm_models(self):
+        return self.client.llms.list()
+
+    def list_embedding_llm_models(self):
+        return self.client.llms.list_embeddings()

codemie_test_harness/tests/utils/search_utils.py

@@ -6,9 +6,14 @@ class SearchUtils(BaseUtils):
     def list_assistants(self, filters):
         return self.client.assistants.list(per_page=100, filters=filters)
 
-    def list_workflows(self, filters=None, projects=None):
+    def list_workflows(self, page=0, per_page=100, filters=None, projects=None):
         return self.client.workflows.list(
-            per_page=100, filters=filters, projects=projects
+            page=page, per_page=per_page, filters=filters, projects=projects
+        )
+
+    def list_workflow_executions(self, test_workflow_id, page=0, per_page=10):
+        return self.client.workflows.executions(test_workflow_id).list(
+            page=page, per_page=per_page
         )
 
     def list_data_sources(
@@ -18,18 +23,19 @@ class SearchUtils(BaseUtils):
         owner=None,
         status=None,
         filters=None,
+        per_page=100,
     ):
         return self.client.datasources.list(
-            per_page=100,
             filters=filters,
             owner=owner,
             status=status,
             datasource_types=datasource_types,
             projects=projects,
+            per_page=per_page,
         )
 
-    def list_integrations(self, setting_type=None, filters=None):
+    def list_integrations(self, page=0, per_page=100, setting_type=None, filters=None):
         setting_type = IntegrationType.USER if setting_type is None else setting_type
         return self.client.integrations.list(
-            per_page=100, setting_type=setting_type, filters=filters
+            page=page, per_page=per_page, setting_type=setting_type, filters=filters
         )

codemie_test_harness/tests/utils/user_utils.py

@@ -0,0 +1,9 @@
+from codemie_test_harness.tests.utils.base_utils import BaseUtils
+
+
+class UserUtils(BaseUtils):
+    def get_about_me(self):
+        return self.client.users.about_me()
+
+    def get_user_data(self):
+        return self.client.users.get_data()

codemie_test_harness/tests/utils/workflow_utils.py

@@ -50,6 +50,7 @@ class WorkflowUtils(BaseUtils):
         self,
         workflow_yaml,
         workflow_type=WorkflowMode.SEQUENTIAL,
+        description=None,
         workflow_name=None,
         shared=False,
         project=None,
@@ -58,7 +59,7 @@ class WorkflowUtils(BaseUtils):
 
         request = WorkflowCreateRequest(
             name=workflow_name,
-            description="Test Workflow",
+            description=description if description else "Test Workflow",
             project=project if project else PROJECT,
             mode=workflow_type,
             yaml_config=workflow_yaml,
@@ -73,6 +74,7 @@ class WorkflowUtils(BaseUtils):
         self,
         workflow_type,
         workflow_yaml,
+        description=None,
         workflow_name=None,
         shared=False,
         project_name=None,
@@ -81,7 +83,12 @@ class WorkflowUtils(BaseUtils):
         Sends request to workflow creation endpoint and waits for workflow created.
         """
         response = self.send_create_workflow_request(
-            workflow_yaml, workflow_type, workflow_name, shared, project_name
+            workflow_yaml,
+            workflow_type,
+            description,
+            workflow_name,
+            shared,
+            project_name,
         )
 
         return wait_for_entity(
@@ -104,28 +111,49 @@ class WorkflowUtils(BaseUtils):
         wait_for_completion(execution_state_service=states_service, state_id=state.id)
         return states_service.get_output(state_id=state.id).output
 
+    def run_workflow(self, workflow_id, user_input=""):
+        return self.client.workflows.run(workflow_id=workflow_id, user_input=user_input)
+
+    def get_workflow(self, workflow_id):
+        return self.client.workflows.get(workflow_id)
+
+    def get_workflow_execution(self, test_workflow_id, execution_id):
+        return self.client.workflows.executions(test_workflow_id).get(execution_id)
+
+    def get_workflow_executions_states(self, test_workflow_id, execution_id):
+        return self.client.workflows.executions(test_workflow_id).states(execution_id)
+
+    def send_update_workflow_request(
+        self,
+        workflow_id,
+        request: WorkflowUpdateRequest,
+    ):
+        return self.client.workflows.update(workflow_id, request)
+
     def update_workflow(
         self,
         workflow,
         name=None,
         description=None,
+        project=None,
         yaml_config=None,
         mode=None,
         shared=None,
     ):
+        name = name if name else workflow.name
         request = WorkflowUpdateRequest(
-            name=name if name else workflow.name,
-            project=workflow.project,
+            name=name,
+            project=project if project else workflow.project,
             description=description if description else workflow.description,
             yaml_config=yaml_config if yaml_config else workflow.yaml_config,
             mode=mode if mode else workflow.mode,
             shared=shared if shared else workflow.shared,
         )
-        self.client.workflows.update(workflow.id, request)
+        self.send_update_workflow_request(workflow.id, request=request)
 
         return wait_for_entity(
             lambda: self.client.workflows.list(per_page=200),
-            entity_name=workflow.name,
+            entity_name=name,
         )
 
     def get_prebuilt_workflows(self):

{codemie_test_harness-0.1.136.dist-info → codemie_test_harness-0.1.137.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.136
+Version: 0.1.137
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.136)
+Requires-Dist: codemie-sdk-python (==0.1.137)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)