codemie-test-harness 0.1.184__py3-none-any.whl → 0.1.198__py3-none-any.whl

codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py

@@ -3,6 +3,7 @@ import os
 import pytest
 
 from codemie_test_harness.tests.enums.tools import PluginTool
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.plugin_tools_test_data import (
     list_files_plugin_tools_test_data,
     CREATE_READ_DELETE_FILE_TEST_DATA,
@@ -37,6 +38,10 @@ def test_workflow_with_list_files_plugin_tools(
     response = workflow_utils.execute_workflow(
         workflow.id, assistant_and_state_name, user_input=prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(workflow)
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -71,6 +76,11 @@ def test_workflow_with_modify_files_plugin_tools(
             ),
         )
 
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+        assert_tool_triggered(PluginTool.WRITE_FILE_TO_FILE_SYSTEM, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["create_file_response"].format(
@@ -86,6 +96,11 @@ def test_workflow_with_modify_files_plugin_tools(
             ),
         )
 
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+        assert_tool_triggered(PluginTool.GENERIC_GIT_TOOL, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["git_command_response"].format(file_name),
@@ -99,6 +114,11 @@ def test_workflow_with_modify_files_plugin_tools(
             ].format(file_name),
         )
 
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+        assert_tool_triggered(PluginTool.READ_FILE_FROM_FILE_SYSTEM, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["show_file_content_response"].format(

codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py

@@ -2,7 +2,8 @@ import os
 
 import pytest
 
-from codemie_test_harness.tests.enums.tools import PluginTool
+from codemie_test_harness.tests.enums.tools import PluginTool, CliMcpServer
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     cli_mcp_server_with_plugin_test_data,
     filesystem_mcp_server_with_plugin_test_data,
@@ -40,6 +41,10 @@ def test_workflow_with_plugin_and_cli_mcp_server(
     response = workflow_utils.execute_workflow(
         workflow.id, assistant_and_state_name, user_input=prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(workflow)
+    assert_tool_triggered(CliMcpServer.RUN_COMMAND, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -74,6 +79,14 @@ def test_workflow_with_plugin_and_filesystem_mcp_server(
         response = workflow_utils.execute_workflow(
             workflow.id, assistant_and_state_name, user_input=prompt
         )
+
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+
+        if tool_name != PluginTool.READ_FILE:
+            assert_tool_triggered(tool_name, triggered_tools)
+
         similarity_check.check_similarity(response, expected_response)
     finally:
         file_to_remove = f"{str(TESTS_PATH / 'sdk_tests')}.properties"

codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py

@@ -3,7 +3,10 @@ import pytest
 from codemie_test_harness.tests.test_data.project_management_test_data import (
     pm_tools_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import (
     project_management_integrations,
 )
@@ -39,4 +42,10 @@ def test_workflow_with_virtual_assistant_with_project_management_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py

@@ -3,7 +3,10 @@ import pytest
 from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
     rp_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.workflow
@@ -36,4 +39,10 @@ def test_workflow_with_virtual_assistant_with_report_portal_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py

@@ -9,6 +9,7 @@ from codemie_test_harness.tests.test_data.research_tools_test_data import (
 from codemie_test_harness.tests.utils.base_utils import (
     get_random_name,
     percent_of_relevant_titles,
+    assert_tool_triggered,
 )
 
 
@@ -38,6 +39,10 @@ def test_workflow_with_search_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
     percent = percent_of_relevant_titles(response)
 
     assert_that(
@@ -74,5 +79,9 @@ def test_workflow_with_interaction_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py

@@ -5,7 +5,10 @@ from codemie_test_harness.tests.test_data.servicenow_tools_test_data import (
     PROMPT,
     EXPECTED_RESPONSE,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.workflow
@@ -29,4 +32,10 @@ def test_workflow_with_virtual_assistant_with_servicenow_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(ServiceNowTool.SERVICE_NOW, triggered_tools)
+
     similarity_check.check_similarity(response, EXPECTED_RESPONSE, 80)

codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py

@@ -3,7 +3,10 @@ import pytest
 from codemie_test_harness.tests.test_data.vcs_tools_test_data import (
     vcs_tools_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import vcs_integrations
 
 
@@ -37,4 +40,9 @@ def test_workflow_with_vcs_tool(
         test_workflow.id, assistant_and_state_name, user_input=prompt
     )
 
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)

{codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.184
+Version: 0.1.198
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,11 +13,13 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.184)
+Requires-Dist: codemie-sdk-python (==0.1.198)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
+Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)
 Requires-Dist: pytest-rerunfailures (>=15.1,<16.0)
+Requires-Dist: pytest-timeout (>=2.4.0,<3.0.0)
 Requires-Dist: pytest-xdist (>=3.6.1,<4.0.0)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
 Requires-Dist: python-gitlab (>=5.6.0,<6.0.0)
@@ -96,6 +98,7 @@ Optional defaults for pytest:
 codemie-test-harness config set PYTEST_MARKS "smoke"
 codemie-test-harness config set PYTEST_N 8
 codemie-test-harness config set PYTEST_RERUNS 2
+codemie-test-harness config set PYTEST_COUNT 10  # For performance testing (optional)
 ```
 
 #### Integration Categories & Management
@@ -293,6 +296,80 @@ codemie-test-harness --git-env github run --marks git
 codemie-test-harness run --marks ui --headless
 ```
 
+#### Performance and Load Testing
+
+Run tests multiple times in parallel to simulate load and measure performance:
+
+```shell
+# Performance test: Run test 50 times with 10 parallel workers
+codemie-test-harness run --marks excel_generation --count 50 -n 10
+
+# Heavy load test: 100 iterations with 20 workers
+codemie-test-harness run --marks excel_generation --count 100 -n 20 -v
+
+# Light load test with retries for stability
+codemie-test-harness run --marks smoke --count 25 -n 5 --reruns 2
+
+# Set default count in config for repeated use
+codemie-test-harness config set PYTEST_COUNT 30
+codemie-test-harness run --marks excel_generation -n 10  # Uses count=30 from config
+
+# Override config default for a specific run
+codemie-test-harness run --marks excel_generation --count 100 -n 20  # Overrides config
+```
+
+**Note:** The `--count` parameter requires the `pytest-repeat` plugin, which is included in the dependencies.
+
+#### Test Timeout Control
+
+Control per-test timeout to prevent tests from running indefinitely. Tests exceeding the timeout will be **automatically terminated and marked as FAILED**.
+
+**Configuration Priority**: CLI args → Environment variable → Config file → Default (300 seconds)
+
+**Usage Examples:**
+
+```shell
+# Via CLI argument (600 seconds = 10 minutes)
+codemie-test-harness run --timeout 600 --marks smoke
+
+# Via config file (persistent)
+codemie-test-harness config set TEST_TIMEOUT 900
+codemie-test-harness run --marks api
+
+# Via environment variable
+export TEST_TIMEOUT=300
+codemie-test-harness run --marks smoke
+
+# Disable timeout for debugging (use 0)
+codemie-test-harness run --timeout 0 --marks smoke
+
+# Override config default for specific run
+codemie-test-harness config set TEST_TIMEOUT 600
+codemie-test-harness run --timeout 1200 --marks slow  # Uses 1200, not 600
+```
+
+**Default**: 300 seconds (5 minutes) per test
+
+**What Happens on Timeout?**
+
+When a test exceeds the configured timeout:
+1. ✅ **Test is automatically terminated** - Execution stops immediately
+2. ✅ **Marked as FAILED** - Test result shows as failed with clear timeout message
+3. ✅ **Error details displayed** - Shows which test timed out and the configured limit
+4. ✅ **Remaining tests continue** - Other tests proceed normally
+5. ✅ **Stack trace captured** - Shows where the test was when timeout occurred
+
+**Example timeout error output:**
+```
+FAILED tests/test_slow_operation.py::test_data_processing - Failed: Timeout >300.0s
+```
+
+**Notes:**
+- Timeout applies to **individual test functions**, not the entire test run
+- Useful for preventing hanging tests in CI/CD pipelines
+- Consider increasing timeout for legitimate long-running operations (data processing, large file operations)
+- Timeout of 0 disables the timeout (use for debugging only)
+
 #### Assistant Chat Interface
 
 Interact directly with CodeMie assistants through the CLI:
@@ -368,6 +445,9 @@ codemie-test-harness config vars data-management
 
 # Test execution with multiple overrides
 codemie-test-harness run --marks "smoke and not ui" -n 10 --reruns 3 --headless
+
+# Performance testing with count parameter
+codemie-test-harness run --marks excel_generation --count 50 -n 10
 ```
 
 #### Common Test Markers
@@ -508,9 +588,60 @@ pytest -m not_for_parallel_run --reruns 2
 # API tests
 pytest -n 10 -m "api and not not_for_parallel_run" --reruns 2
 pytest -m not_for_parallel_run --reruns 3
+
+# Performance/Load testing: Run test multiple times in parallel
+pytest -n 10 --count 50 -m excel_generation  # Run 50 times with 10 workers
+pytest -n 20 --count 100 -m smoke --reruns 2  # Heavy load with retries
+```
+
+**Notes:**
+- `--reruns 2` uses pytest-rerunfailures to improve resiliency in flaky environments
+- `--count N` uses pytest-repeat to run each test N times (useful for performance/load testing)
+
+#### Test Timeout Configuration
+
+Tests have a configurable timeout to prevent hanging. Default is **300 seconds (5 minutes)** per test.
+
+**Configure in .env file:**
+```properties
+TEST_TIMEOUT=600  # 10 minutes
+```
+
+**Override via pytest CLI:**
+```shell
+# Set timeout for this run
+pytest -n 10 -m smoke --timeout 900  # 15 minutes
+
+# Disable timeout (debugging)
+pytest -m slow_tests --timeout 0
+
+# Use default from .env or pytest.ini
+pytest -n 10 -m api  # Uses TEST_TIMEOUT from .env or 300s default
+```
+
+**Timeout Behavior:**
+
+When a test exceeds the configured timeout:
+- Test execution is **terminated immediately**
+- Test is marked as **FAILED** with a timeout error message
+- Stack trace shows where the test was when timeout occurred
+- Remaining tests continue execution normally
+
+**Example timeout failure:**
+```
+================================== FAILURES ===================================
+_________________ test_slow_workflow_execution ________________
+
+E   Failed: Timeout >300.0s
+
+tests/workflow/test_workflows.py:145: Failed
 ```
 
-Note: "--reruns 2" uses pytest-rerunfailures to improve resiliency in flaky environments.
+**Best Practices:**
+- Keep default timeout reasonable (5-10 minutes for E2E tests)
+- Increase timeout for specific slow tests using `@pytest.mark.timeout(900)`
+- Use timeout=0 only for debugging hanging tests
+- Consider if a test legitimately needs > 5 minutes (optimize if possible)
 
 ### UI tests (Playwright)