codemie-test-harness 0.1.200__tar.gz → 0.1.202__tar.gz

{codemie_test_harness-0.1.200 → codemie_test_harness-0.1.202}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.200
+Version: 0.1.202
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.200)
+Requires-Dist: codemie-sdk-python (==0.1.202)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)
@@ -59,6 +59,7 @@ The CLI now provides four main command groups:
 2. **`run`** - Enhanced test execution with flexible parameters
 3. **`assistant`** - Direct assistant interaction and chat capabilities
 4. **`workflow`** - Workflow execution
+5. **`marks`** - List all available pytest marks in the test suite.
 
 Each command group includes extensive help and validation features.
 
@@ -101,6 +102,49 @@ codemie-test-harness config set PYTEST_RERUNS 2
 codemie-test-harness config set PYTEST_COUNT 10  # For performance testing (optional)
 ```
 
+#### Running Tests Locally (Minimal Configuration)
+
+If you're running tests against a **local CodeMie instance** and want to use integration settings already stored in AWS Parameter Store (instead of configuring each integration manually), you only need to set these **4 minimal keys**:
+
+```shell
+# Minimal local setup - uses AWS Parameter Store for all integrations
+codemie-test-harness config set AWS_ACCESS_KEY <your_aws_access_key>
+codemie-test-harness config set AWS_SECRET_KEY <your_aws_secret_key>
+codemie-test-harness config set CODEMIE_API_DOMAIN http://localhost:8080
+codemie-test-harness config set TEST_USER_FULL_NAME "dev-codemie-user"
+```
+
+Then run your tests:
+
+```shell
+# Run all tests locally
+codemie-test-harness run
+
+# Run specific test categories
+codemie-test-harness run --marks smoke
+codemie-test-harness run --marks "api and not not_for_parallel_run" -n 8
+```
+
+**How it works:**
+- The AWS credentials allow the test harness to automatically fetch integration credentials (GitLab, JIRA, Confluence, etc.) from AWS Parameter Store
+- You don't need to manually configure individual integrations unless you want to override specific values
+- All 86+ integration variables are automatically pulled from Parameter Store as needed
+- This is ideal for local development and testing against your local CodeMie backend
+
+**To override specific integrations locally:**
+
+If you need to use your own tokens instead of shared Parameter Store values:
+
+```shell
+# Override with personal GitLab token
+codemie-test-harness config set GITLAB_TOKEN <your_personal_token>
+
+# Override with personal JIRA credentials
+codemie-test-harness config set JIRA_TOKEN <your_personal_token>
+```
+
+Values set explicitly in the config take priority over AWS Parameter Store values.
+
 #### Integration Categories & Management
 
 The CLI supports **10 major integration categories** with comprehensive credential management:
@@ -296,6 +340,35 @@ codemie-test-harness --git-env github run --marks git
 codemie-test-harness run --marks ui --headless
 ```
 
+#### Advanced Marks Usage (Logical Operators)
+
+Combine multiple markers using `and`, `or`, and `not` keywords for fine-grained test selection:
+
+```shell
+# OR operator - run tests with either marker
+codemie-test-harness run --marks "smoke or gitlab" -n 8
+codemie-test-harness run --marks "jira_kb or confluence_kb" -n 6
+
+# AND operator - run tests with both markers
+codemie-test-harness run --marks "api and smoke" -n 10
+codemie-test-harness run --marks "gitlab and code_kb" -n 4
+
+# NOT operator - exclude specific markers
+codemie-test-harness run --marks "api and not ui" -n 10
+codemie-test-harness run --marks "not not_for_parallel_run" -n 12
+
+# Complex combinations with parentheses
+codemie-test-harness run --marks "(smoke or api) and not ui" -n 8
+codemie-test-harness run --marks "(gitlab or github) and not not_for_parallel_run" -n 10
+codemie-test-harness run --marks "smoke and (jira_kb or confluence_kb)" -n 6
+
+# Exclude multiple markers
+codemie-test-harness run --marks "api and not (ui or not_for_parallel_run)" -n 10
+
+# Run all knowledge base tests except code
+codemie-test-harness run --marks "(jira_kb or confluence_kb) and not code_kb" -n 8
+```
+
 #### Performance and Load Testing
 
 Run tests multiple times in parallel to simulate load and measure performance:
@@ -542,7 +615,7 @@ The credentials manager supports **86+ environment variables** across **10 categ
 2) Create a .env file in the project root. If you provide AWS credentials, the suite will fetch additional values from AWS Systems Manager Parameter Store and recreate .env accordingly.
 
 ```properties
-ENV=local
+CODEMIE_API_DOMAIN=http://localhost:8080
 
 AWS_ACCESS_KEY=<aws_access_token>
 AWS_SECRET_KEY=<aws_secret_key>
@@ -594,9 +667,42 @@ pytest -n 10 --count 50 -m excel_generation  # Run 50 times with 10 workers
 pytest -n 20 --count 100 -m smoke --reruns 2  # Heavy load with retries
 ```
 
+**Advanced Marks Usage with Logical Operators:**
+
+Combine markers using `and`, `or`, and `not` for precise test selection:
+
+```shell
+# OR operator - run tests with either marker
+pytest -n 8 -m "smoke or gitlab" --reruns 2
+pytest -n 6 -m "jira_kb or confluence_kb" --reruns 2
+
+# AND operator - run tests with both markers
+pytest -n 10 -m "api and smoke" --reruns 2
+pytest -n 4 -m "gitlab and code_kb" --reruns 2
+
+# NOT operator - exclude specific markers
+pytest -n 10 -m "api and not ui" --reruns 2
+pytest -n 12 -m "not not_for_parallel_run" --reruns 2
+
+# Complex combinations with parentheses
+pytest -n 8 -m "(smoke or api) and not ui" --reruns 2
+pytest -n 10 -m "(gitlab or github) and not not_for_parallel_run" --reruns 2
+pytest -n 6 -m "smoke and (jira_kb or confluence_kb)" --reruns 2
+
+# Exclude multiple markers
+pytest -n 10 -m "api and not (ui or not_for_parallel_run)" --reruns 2
+
+# Run all knowledge base tests except code KB
+pytest -n 8 -m "(jira_kb or confluence_kb) and not code_kb" --reruns 2
+
+# Run all Git-related tests (GitLab or GitHub)
+pytest -n 8 -m "gitlab or github or git" --reruns 2
+```
+
 **Notes:**
 - `--reruns 2` uses pytest-rerunfailures to improve resiliency in flaky environments
 - `--count N` uses pytest-repeat to run each test N times (useful for performance/load testing)
+- Use quotes around marker expressions containing spaces or special characters
 
 #### Test Timeout Configuration
 

{codemie_test_harness-0.1.200 → codemie_test_harness-0.1.202}/README.md

@@ -30,6 +30,7 @@ The CLI now provides four main command groups:
 2. **`run`** - Enhanced test execution with flexible parameters
 3. **`assistant`** - Direct assistant interaction and chat capabilities
 4. **`workflow`** - Workflow execution
+5. **`marks`** - List all available pytest marks in the test suite.
 
 Each command group includes extensive help and validation features.
 
@@ -72,6 +73,49 @@ codemie-test-harness config set PYTEST_RERUNS 2
 codemie-test-harness config set PYTEST_COUNT 10  # For performance testing (optional)
 ```
 
+#### Running Tests Locally (Minimal Configuration)
+
+If you're running tests against a **local CodeMie instance** and want to use integration settings already stored in AWS Parameter Store (instead of configuring each integration manually), you only need to set these **4 minimal keys**:
+
+```shell
+# Minimal local setup - uses AWS Parameter Store for all integrations
+codemie-test-harness config set AWS_ACCESS_KEY <your_aws_access_key>
+codemie-test-harness config set AWS_SECRET_KEY <your_aws_secret_key>
+codemie-test-harness config set CODEMIE_API_DOMAIN http://localhost:8080
+codemie-test-harness config set TEST_USER_FULL_NAME "dev-codemie-user"
+```
+
+Then run your tests:
+
+```shell
+# Run all tests locally
+codemie-test-harness run
+
+# Run specific test categories
+codemie-test-harness run --marks smoke
+codemie-test-harness run --marks "api and not not_for_parallel_run" -n 8
+```
+
+**How it works:**
+- The AWS credentials allow the test harness to automatically fetch integration credentials (GitLab, JIRA, Confluence, etc.) from AWS Parameter Store
+- You don't need to manually configure individual integrations unless you want to override specific values
+- All 86+ integration variables are automatically pulled from Parameter Store as needed
+- This is ideal for local development and testing against your local CodeMie backend
+
+**To override specific integrations locally:**
+
+If you need to use your own tokens instead of shared Parameter Store values:
+
+```shell
+# Override with personal GitLab token
+codemie-test-harness config set GITLAB_TOKEN <your_personal_token>
+
+# Override with personal JIRA credentials
+codemie-test-harness config set JIRA_TOKEN <your_personal_token>
+```
+
+Values set explicitly in the config take priority over AWS Parameter Store values.
+
 #### Integration Categories & Management
 
 The CLI supports **10 major integration categories** with comprehensive credential management:
@@ -267,6 +311,35 @@ codemie-test-harness --git-env github run --marks git
 codemie-test-harness run --marks ui --headless
 ```
 
+#### Advanced Marks Usage (Logical Operators)
+
+Combine multiple markers using `and`, `or`, and `not` keywords for fine-grained test selection:
+
+```shell
+# OR operator - run tests with either marker
+codemie-test-harness run --marks "smoke or gitlab" -n 8
+codemie-test-harness run --marks "jira_kb or confluence_kb" -n 6
+
+# AND operator - run tests with both markers
+codemie-test-harness run --marks "api and smoke" -n 10
+codemie-test-harness run --marks "gitlab and code_kb" -n 4
+
+# NOT operator - exclude specific markers
+codemie-test-harness run --marks "api and not ui" -n 10
+codemie-test-harness run --marks "not not_for_parallel_run" -n 12
+
+# Complex combinations with parentheses
+codemie-test-harness run --marks "(smoke or api) and not ui" -n 8
+codemie-test-harness run --marks "(gitlab or github) and not not_for_parallel_run" -n 10
+codemie-test-harness run --marks "smoke and (jira_kb or confluence_kb)" -n 6
+
+# Exclude multiple markers
+codemie-test-harness run --marks "api and not (ui or not_for_parallel_run)" -n 10
+
+# Run all knowledge base tests except code
+codemie-test-harness run --marks "(jira_kb or confluence_kb) and not code_kb" -n 8
+```
+
 #### Performance and Load Testing
 
 Run tests multiple times in parallel to simulate load and measure performance:
@@ -513,7 +586,7 @@ The credentials manager supports **86+ environment variables** across **10 categ
 2) Create a .env file in the project root. If you provide AWS credentials, the suite will fetch additional values from AWS Systems Manager Parameter Store and recreate .env accordingly.
 
 ```properties
-ENV=local
+CODEMIE_API_DOMAIN=http://localhost:8080
 
 AWS_ACCESS_KEY=<aws_access_token>
 AWS_SECRET_KEY=<aws_secret_key>
@@ -565,9 +638,42 @@ pytest -n 10 --count 50 -m excel_generation  # Run 50 times with 10 workers
 pytest -n 20 --count 100 -m smoke --reruns 2  # Heavy load with retries
 ```
 
+**Advanced Marks Usage with Logical Operators:**
+
+Combine markers using `and`, `or`, and `not` for precise test selection:
+
+```shell
+# OR operator - run tests with either marker
+pytest -n 8 -m "smoke or gitlab" --reruns 2
+pytest -n 6 -m "jira_kb or confluence_kb" --reruns 2
+
+# AND operator - run tests with both markers
+pytest -n 10 -m "api and smoke" --reruns 2
+pytest -n 4 -m "gitlab and code_kb" --reruns 2
+
+# NOT operator - exclude specific markers
+pytest -n 10 -m "api and not ui" --reruns 2
+pytest -n 12 -m "not not_for_parallel_run" --reruns 2
+
+# Complex combinations with parentheses
+pytest -n 8 -m "(smoke or api) and not ui" --reruns 2
+pytest -n 10 -m "(gitlab or github) and not not_for_parallel_run" --reruns 2
+pytest -n 6 -m "smoke and (jira_kb or confluence_kb)" --reruns 2
+
+# Exclude multiple markers
+pytest -n 10 -m "api and not (ui or not_for_parallel_run)" --reruns 2
+
+# Run all knowledge base tests except code KB
+pytest -n 8 -m "(jira_kb or confluence_kb) and not code_kb" --reruns 2
+
+# Run all Git-related tests (GitLab or GitHub)
+pytest -n 8 -m "gitlab or github or git" --reruns 2
+```
+
 **Notes:**
 - `--reruns 2` uses pytest-rerunfailures to improve resiliency in flaky environments
 - `--count N` uses pytest-repeat to run each test N times (useful for performance/load testing)
+- Use quotes around marker expressions containing spaces or special characters
 
 #### Test Timeout Configuration
 

{codemie_test_harness-0.1.200 → codemie_test_harness-0.1.202}/codemie_test_harness/tests/utils/workflow_utils.py

@@ -1,10 +1,13 @@
 import os
+from pathlib import Path
+from typing import Optional
 
 from codemie_sdk.models.workflow import (
     WorkflowCreateRequest,
     WorkflowUpdateRequest,
     WorkflowMode,
 )
+
 from codemie_test_harness.tests import PROJECT
 from codemie_test_harness.tests.utils.base_utils import (
     BaseUtils,
@@ -97,8 +100,14 @@ class WorkflowUtils(BaseUtils):
             entity_name=response[1],
         )
 
-    def execute_workflow(self, workflow, execution_name, user_input=""):
-        self.client.workflows.run(workflow, user_input=user_input)
+    def execute_workflow(
+        self,
+        workflow,
+        execution_name,
+        user_input="",
+        file_name: Optional[str] = None,
+    ):
+        self.client.workflows.run(workflow, user_input=user_input, file_name=file_name)
         executions = self.client.workflows.executions(workflow)
         execution_id = next(
             row.execution_id for row in executions.list() if row.prompt == user_input
@@ -232,6 +241,9 @@ class WorkflowUtils(BaseUtils):
 
         return triggered_tools
 
+    def upload_file(self, file_path: Path):
+        return self.client.files.bulk_upload([file_path])
+
     @staticmethod
     def _extract_tools_from_thoughts(thoughts_data):
         """

codemie_test_harness-0.1.202/codemie_test_harness/tests/workflow/test_workflows.py

@@ -0,0 +1,237 @@
+import json
+
+import pytest
+from hamcrest import assert_that, equal_to
+
+from codemie_test_harness.tests.enums.tools import Default
+from codemie_test_harness.tests.test_data.assistant_test_data import (
+    EXCEL_TOOL_TEST_DATA,
+    DOCX_TOOL_TEST_DATA,
+)
+from codemie_test_harness.tests.test_data.file_test_data import file_test_data
+from codemie_test_harness.tests.test_data.output_schema_test_data import output_schema
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.utils.constants import FILES_PATH
+from codemie_test_harness.tests.utils.yaml_utils import AssistantModel, StateModel
+
+
+@pytest.mark.workflow
+@pytest.mark.api
+@pytest.mark.smoke
+def test_workflow_with_json_output_schema(default_llm, workflow, workflow_utils):
+    assistant_and_state_name = get_random_name()
+
+    assistant = AssistantModel(
+        id=assistant_and_state_name,
+        model=default_llm.base_name,
+        system_prompt="You are a helpful assistant.",
+    )
+
+    state = StateModel(
+        id=assistant_and_state_name,
+        assistant_id=assistant_and_state_name,
+        output_schema=json.dumps(output_schema),
+    )
+
+    workflow = workflow(
+        workflow_name=get_random_name(),
+        assistant_model=assistant,
+        state_model=state,
+    )
+
+    response = workflow_utils.execute_workflow(
+        workflow.id, assistant_and_state_name, user_input="1+1?"
+    )
+
+    assert_that(json.loads(response)["results"][0], equal_to(2))
+
+
+@pytest.mark.workflow
+@pytest.mark.api
+@pytest.mark.smoke
+@pytest.mark.file
+@pytest.mark.parametrize(
+    "file_name, expected_response, expected_tool",
+    file_test_data,
+    ids=[f"{row[0]}" for row in file_test_data],
+)
+def test_workflow_with_user_input_and_file_attachment(
+    workflow_with_virtual_assistant,
+    workflow_utils,
+    assistant_utils,
+    file_name,
+    expected_response,
+    expected_tool,
+    similarity_check,
+):
+    """
+    Test workflow execution with user input that includes file attachment.
+
+    This test demonstrates how workflows can handle file attachments by:
+    1. Uploading a file
+    2. Creating a workflow with file processing capabilities
+    3. Executing the workflow with user input that references the uploaded file
+    4. Verifying that file processing tools are triggered
+    """
+    assistant_and_state_name = get_random_name()
+
+    # Upload file to get file URL
+    upload_response = workflow_utils.upload_file(FILES_PATH / file_name)
+    file_url = upload_response.files[0].file_url
+
+    # Create workflow with virtual assistant that has file analysis capabilities
+    # Note: File analysis tools are automatically available when files are processed
+    system_prompt = "You are a helpful assistant that can analyze and process files. "
+
+    workflow_instance = workflow_with_virtual_assistant(
+        assistant_and_state_name=assistant_and_state_name,
+        system_prompt=system_prompt,
+        task=(
+            "Analyze the uploaded file from the provided file URL and give a detailed summary. "
+            "Use the appropriate file analysis tools based on the file type to extract and process the content."
+        ),
+    )
+
+    # Prepare user input that includes file reference
+    user_input = "Please provide a summary about file content."
+
+    # Execute workflow
+    response = workflow_utils.execute_workflow(
+        workflow_instance.id,
+        assistant_and_state_name,
+        user_input=user_input,
+        file_name=file_url,
+    )
+
+    # Extract triggered tools from execution
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        workflow_instance
+    )
+
+    assert_tool_triggered(expected_tool, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
+
+
+@pytest.mark.workflow
+@pytest.mark.file
+@pytest.mark.api
+@pytest.mark.smoke
+@pytest.mark.parametrize("prompt, expected_response", EXCEL_TOOL_TEST_DATA)
+def test_workflow_excel_tool_extended_functionality(
+    workflow_with_virtual_assistant,
+    workflow_utils,
+    similarity_check,
+    prompt,
+    expected_response,
+):
+    """
+    Test extended Excel tool functionality with various scenarios in workflow.
+
+    This test covers:
+    - Data extraction from visible sheets only
+    - All data including hidden sheets
+    - Sheet name listing functionality
+    - File statistics and structure analysis
+    - Single sheet extraction by index and name
+    - Data cleaning and normalization
+    - Hidden sheet visibility control
+    - Column structure and data type analysis
+    - Tabular structure normalization
+    - Multi-sheet comprehensive analysis
+
+    """
+    assistant_and_state_name = get_random_name()
+
+    # Upload file to get file URL
+    upload_response = workflow_utils.upload_file(FILES_PATH / "test_extended.xlsx")
+    file_url = upload_response.files[0].file_url
+
+    # Create workflow with virtual assistant that has Excel file processing capabilities
+    system_prompt = "You have all required information in initial prompt. Do not ask additional questions and proceed with request."
+
+    workflow_instance = workflow_with_virtual_assistant(
+        assistant_and_state_name=assistant_and_state_name,
+        system_prompt=system_prompt,
+        task=prompt,
+    )
+
+    # Execute workflow with the file URL
+    response = workflow_utils.execute_workflow(
+        workflow_instance.id,
+        assistant_and_state_name,
+        user_input="Process the uploaded Excel file as requested.",
+        file_name=file_url,
+    )
+
+    # Extract triggered tools from execution
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        workflow_instance
+    )
+
+    assert_tool_triggered(Default.EXCEL_TOOL, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
+
+
+@pytest.mark.workflow
+@pytest.mark.file
+@pytest.mark.api
+@pytest.mark.smoke
+@pytest.mark.parametrize("prompt, expected_response", DOCX_TOOL_TEST_DATA)
+def test_workflow_docx_tool_extended_functionality(
+    workflow_with_virtual_assistant,
+    workflow_utils,
+    similarity_check,
+    prompt,
+    expected_response,
+):
+    """
+    Test extended Docx tool functionality with various scenarios in workflow.
+
+    This test covers:
+    - Extract plain text using 'text' query
+    - Extract text with metadata using 'text_with_metadata' query
+    - Extract document structure using 'structure_only' query
+    - Extract tables using 'table_extraction' query
+    - Generate summary using 'summary' query
+    - Perform analysis with custom instructions using 'analyze' query
+    - Process specific pages '1-3' using pages parameter
+    - Process specific pages '1,5,10' using pages parameter
+    - Extract images using 'image_extraction' query
+    - Extract text with OCR from images using 'text_with_images' query
+
+    """
+    assistant_and_state_name = get_random_name()
+
+    # Upload file to get file URL
+    upload_response = workflow_utils.upload_file(FILES_PATH / "test_extended.docx")
+    file_url = upload_response.files[0].file_url
+
+    # Create workflow with virtual assistant that has DOCX file processing capabilities
+    system_prompt = """You are a helpful assistant that can analyze and process DOCX files.
+                       You have all required information in initial prompt.
+                       Do not ask additional questions and proceed with request."""
+
+    workflow_instance = workflow_with_virtual_assistant(
+        assistant_and_state_name=assistant_and_state_name,
+        system_prompt=system_prompt,
+        task=prompt,
+    )
+
+    # Execute workflow with the file URL
+    response = workflow_utils.execute_workflow(
+        workflow_instance.id,
+        assistant_and_state_name,
+        user_input="Process the uploaded DOCX file as requested.",
+        file_name=file_url,
+    )
+
+    # Extract triggered tools from execution
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        workflow_instance
+    )
+
+    assert_tool_triggered(Default.DOCX_TOOL, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)

{codemie_test_harness-0.1.200 → codemie_test_harness-0.1.202}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemie-test-harness"
-version = "0.1.200"
+version = "0.1.202"
 description = "Autotest for CodeMie backend and UI"
 authors = [
     "Anton Yeromin <anton_yeromin@epam.com>",
@@ -12,7 +12,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.12"
-codemie-sdk-python = "0.1.200"
+codemie-sdk-python = "0.1.202"
 pytest = "^8.4.1"
 python-dotenv = "^1.1.0"
 PyHamcrest = "^2.1.0"

codemie_test_harness-0.1.200/codemie_test_harness/tests/workflow/test_workflows.py

@@ -1,39 +0,0 @@
-import json
-
-import pytest
-from hamcrest import assert_that, equal_to
-
-from codemie_test_harness.tests.test_data.output_schema_test_data import output_schema
-from codemie_test_harness.tests.utils.base_utils import get_random_name
-from codemie_test_harness.tests.utils.yaml_utils import AssistantModel, StateModel
-
-
-@pytest.mark.workflow
-@pytest.mark.api
-@pytest.mark.smoke
-def test_workflow_with_json_output_schema(default_llm, workflow, workflow_utils):
-    assistant_and_state_name = get_random_name()
-
-    assistant = AssistantModel(
-        id=assistant_and_state_name,
-        model=default_llm.base_name,
-        system_prompt="You are a helpful assistant.",
-    )
-
-    state = StateModel(
-        id=assistant_and_state_name,
-        assistant_id=assistant_and_state_name,
-        output_schema=json.dumps(output_schema),
-    )
-
-    workflow = workflow(
-        workflow_name=get_random_name(),
-        assistant_model=assistant,
-        state_model=state,
-    )
-
-    response = workflow_utils.execute_workflow(
-        workflow.id, assistant_and_state_name, user_input="1+1?"
-    )
-
-    assert_that(json.loads(response)["results"][0], equal_to(2))