codemie-test-harness 0.1.184__py3-none-any.whl → 0.1.197__py3-none-any.whl

codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py

@@ -3,6 +3,7 @@ import os
 import pytest
 
 from codemie_test_harness.tests.enums.tools import PluginTool
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.plugin_tools_test_data import (
     list_files_plugin_tools_test_data,
     CREATE_READ_DELETE_FILE_TEST_DATA,
@@ -37,6 +38,10 @@ def test_workflow_with_list_files_plugin_tools(
     response = workflow_utils.execute_workflow(
         workflow.id, assistant_and_state_name, user_input=prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(workflow)
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -71,6 +76,11 @@ def test_workflow_with_modify_files_plugin_tools(
             ),
         )
 
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+        assert_tool_triggered(PluginTool.WRITE_FILE_TO_FILE_SYSTEM, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["create_file_response"].format(
@@ -86,6 +96,11 @@ def test_workflow_with_modify_files_plugin_tools(
             ),
         )
 
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+        assert_tool_triggered(PluginTool.GENERIC_GIT_TOOL, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["git_command_response"].format(file_name),
@@ -99,6 +114,11 @@ def test_workflow_with_modify_files_plugin_tools(
             ].format(file_name),
         )
 
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+        assert_tool_triggered(PluginTool.READ_FILE_FROM_FILE_SYSTEM, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["show_file_content_response"].format(

codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py

@@ -2,7 +2,8 @@ import os
 
 import pytest
 
-from codemie_test_harness.tests.enums.tools import PluginTool
+from codemie_test_harness.tests.enums.tools import PluginTool, CliMcpServer
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     cli_mcp_server_with_plugin_test_data,
     filesystem_mcp_server_with_plugin_test_data,
@@ -40,6 +41,10 @@ def test_workflow_with_plugin_and_cli_mcp_server(
     response = workflow_utils.execute_workflow(
         workflow.id, assistant_and_state_name, user_input=prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(workflow)
+    assert_tool_triggered(CliMcpServer.RUN_COMMAND, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -74,6 +79,14 @@ def test_workflow_with_plugin_and_filesystem_mcp_server(
         response = workflow_utils.execute_workflow(
             workflow.id, assistant_and_state_name, user_input=prompt
         )
+
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            workflow
+        )
+
+        if tool_name != PluginTool.READ_FILE:
+            assert_tool_triggered(tool_name, triggered_tools)
+
         similarity_check.check_similarity(response, expected_response)
     finally:
         file_to_remove = f"{str(TESTS_PATH / 'sdk_tests')}.properties"

codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py

@@ -3,7 +3,10 @@ import pytest
 from codemie_test_harness.tests.test_data.project_management_test_data import (
     pm_tools_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import (
     project_management_integrations,
 )
@@ -39,4 +42,10 @@ def test_workflow_with_virtual_assistant_with_project_management_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py

@@ -3,7 +3,10 @@ import pytest
 from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
     rp_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.workflow
@@ -36,4 +39,10 @@ def test_workflow_with_virtual_assistant_with_report_portal_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py

@@ -9,6 +9,7 @@ from codemie_test_harness.tests.test_data.research_tools_test_data import (
 from codemie_test_harness.tests.utils.base_utils import (
     get_random_name,
     percent_of_relevant_titles,
+    assert_tool_triggered,
 )
 
 
@@ -38,6 +39,10 @@ def test_workflow_with_search_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
     percent = percent_of_relevant_titles(response)
 
     assert_that(
@@ -74,5 +79,9 @@ def test_workflow_with_interaction_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py

@@ -5,7 +5,10 @@ from codemie_test_harness.tests.test_data.servicenow_tools_test_data import (
     PROMPT,
     EXPECTED_RESPONSE,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.workflow
@@ -29,4 +32,10 @@ def test_workflow_with_virtual_assistant_with_servicenow_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(ServiceNowTool.SERVICE_NOW, triggered_tools)
+
     similarity_check.check_similarity(response, EXPECTED_RESPONSE, 80)

codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py

@@ -3,7 +3,10 @@ import pytest
 from codemie_test_harness.tests.test_data.vcs_tools_test_data import (
     vcs_tools_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import vcs_integrations
 
 
@@ -37,4 +40,9 @@ def test_workflow_with_vcs_tool(
         test_workflow.id, assistant_and_state_name, user_input=prompt
     )
 
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)

{codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.197.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.184
+Version: 0.1.197
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,9 +13,10 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.184)
+Requires-Dist: codemie-sdk-python (==0.1.197)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
+Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)
 Requires-Dist: pytest-rerunfailures (>=15.1,<16.0)
 Requires-Dist: pytest-xdist (>=3.6.1,<4.0.0)
@@ -96,6 +97,7 @@ Optional defaults for pytest:
 codemie-test-harness config set PYTEST_MARKS "smoke"
 codemie-test-harness config set PYTEST_N 8
 codemie-test-harness config set PYTEST_RERUNS 2
+codemie-test-harness config set PYTEST_COUNT 10  # For performance testing (optional)
 ```
 
 #### Integration Categories & Management
@@ -293,6 +295,30 @@ codemie-test-harness --git-env github run --marks git
 codemie-test-harness run --marks ui --headless
 ```
 
+#### Performance and Load Testing
+
+Run tests multiple times in parallel to simulate load and measure performance:
+
+```shell
+# Performance test: Run test 50 times with 10 parallel workers
+codemie-test-harness run --marks excel_generation --count 50 -n 10
+
+# Heavy load test: 100 iterations with 20 workers
+codemie-test-harness run --marks excel_generation --count 100 -n 20 -v
+
+# Light load test with retries for stability
+codemie-test-harness run --marks smoke --count 25 -n 5 --reruns 2
+
+# Set default count in config for repeated use
+codemie-test-harness config set PYTEST_COUNT 30
+codemie-test-harness run --marks excel_generation -n 10  # Uses count=30 from config
+
+# Override config default for a specific run
+codemie-test-harness run --marks excel_generation --count 100 -n 20  # Overrides config
+```
+
+**Note:** The `--count` parameter requires the `pytest-repeat` plugin, which is included in the dependencies.
+
 #### Assistant Chat Interface
 
 Interact directly with CodeMie assistants through the CLI:
@@ -368,6 +394,9 @@ codemie-test-harness config vars data-management
 
 # Test execution with multiple overrides
 codemie-test-harness run --marks "smoke and not ui" -n 10 --reruns 3 --headless
+
+# Performance testing with count parameter
+codemie-test-harness run --marks excel_generation --count 50 -n 10
 ```
 
 #### Common Test Markers
@@ -508,9 +537,15 @@ pytest -m not_for_parallel_run --reruns 2
 # API tests
 pytest -n 10 -m "api and not not_for_parallel_run" --reruns 2
 pytest -m not_for_parallel_run --reruns 3
+
+# Performance/Load testing: Run test multiple times in parallel
+pytest -n 10 --count 50 -m excel_generation  # Run 50 times with 10 workers
+pytest -n 20 --count 100 -m smoke --reruns 2  # Heavy load with retries
 ```
 
-Note: "--reruns 2" uses pytest-rerunfailures to improve resiliency in flaky environments.
+**Notes:**
+- `--reruns 2` uses pytest-rerunfailures to improve resiliency in flaky environments
+- `--count N` uses pytest-repeat to run each test N times (useful for performance/load testing)
 
 ### UI tests (Playwright)