codemie-test-harness 0.1.183__tar.gz → 0.1.198__tar.gz

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.183
+Version: 0.1.198
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,11 +13,13 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.183)
+Requires-Dist: codemie-sdk-python (==0.1.198)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
+Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)
 Requires-Dist: pytest-rerunfailures (>=15.1,<16.0)
+Requires-Dist: pytest-timeout (>=2.4.0,<3.0.0)
 Requires-Dist: pytest-xdist (>=3.6.1,<4.0.0)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
 Requires-Dist: python-gitlab (>=5.6.0,<6.0.0)
@@ -96,6 +98,7 @@ Optional defaults for pytest:
 codemie-test-harness config set PYTEST_MARKS "smoke"
 codemie-test-harness config set PYTEST_N 8
 codemie-test-harness config set PYTEST_RERUNS 2
+codemie-test-harness config set PYTEST_COUNT 10  # For performance testing (optional)
 ```
 
 #### Integration Categories & Management
@@ -293,6 +296,80 @@ codemie-test-harness --git-env github run --marks git
 codemie-test-harness run --marks ui --headless
 ```
 
+#### Performance and Load Testing
+
+Run tests multiple times in parallel to simulate load and measure performance:
+
+```shell
+# Performance test: Run test 50 times with 10 parallel workers
+codemie-test-harness run --marks excel_generation --count 50 -n 10
+
+# Heavy load test: 100 iterations with 20 workers
+codemie-test-harness run --marks excel_generation --count 100 -n 20 -v
+
+# Light load test with retries for stability
+codemie-test-harness run --marks smoke --count 25 -n 5 --reruns 2
+
+# Set default count in config for repeated use
+codemie-test-harness config set PYTEST_COUNT 30
+codemie-test-harness run --marks excel_generation -n 10  # Uses count=30 from config
+
+# Override config default for a specific run
+codemie-test-harness run --marks excel_generation --count 100 -n 20  # Overrides config
+```
+
+**Note:** The `--count` parameter requires the `pytest-repeat` plugin, which is included in the dependencies.
+
+#### Test Timeout Control
+
+Control per-test timeout to prevent tests from running indefinitely. Tests exceeding the timeout will be **automatically terminated and marked as FAILED**.
+
+**Configuration Priority**: CLI args → Environment variable → Config file → Default (300 seconds)
+
+**Usage Examples:**
+
+```shell
+# Via CLI argument (600 seconds = 10 minutes)
+codemie-test-harness run --timeout 600 --marks smoke
+
+# Via config file (persistent)
+codemie-test-harness config set TEST_TIMEOUT 900
+codemie-test-harness run --marks api
+
+# Via environment variable
+export TEST_TIMEOUT=300
+codemie-test-harness run --marks smoke
+
+# Disable timeout for debugging (use 0)
+codemie-test-harness run --timeout 0 --marks smoke
+
+# Override config default for specific run
+codemie-test-harness config set TEST_TIMEOUT 600
+codemie-test-harness run --timeout 1200 --marks slow  # Uses 1200, not 600
+```
+
+**Default**: 300 seconds (5 minutes) per test
+
+**What Happens on Timeout?**
+
+When a test exceeds the configured timeout:
+1. ✅ **Test is automatically terminated** - Execution stops immediately
+2. ✅ **Marked as FAILED** - Test result shows as failed with clear timeout message
+3. ✅ **Error details displayed** - Shows which test timed out and the configured limit
+4. ✅ **Remaining tests continue** - Other tests proceed normally
+5. ✅ **Stack trace captured** - Shows where the test was when timeout occurred
+
+**Example timeout error output:**
+```
+FAILED tests/test_slow_operation.py::test_data_processing - Failed: Timeout >300.0s
+```
+
+**Notes:**
+- Timeout applies to **individual test functions**, not the entire test run
+- Useful for preventing hanging tests in CI/CD pipelines
+- Consider increasing timeout for legitimate long-running operations (data processing, large file operations)
+- Timeout of 0 disables the timeout (use for debugging only)
+
 #### Assistant Chat Interface
 
 Interact directly with CodeMie assistants through the CLI:
@@ -368,6 +445,9 @@ codemie-test-harness config vars data-management
 
 # Test execution with multiple overrides
 codemie-test-harness run --marks "smoke and not ui" -n 10 --reruns 3 --headless
+
+# Performance testing with count parameter
+codemie-test-harness run --marks excel_generation --count 50 -n 10
 ```
 
 #### Common Test Markers
@@ -508,9 +588,60 @@ pytest -m not_for_parallel_run --reruns 2
 # API tests
 pytest -n 10 -m "api and not not_for_parallel_run" --reruns 2
 pytest -m not_for_parallel_run --reruns 3
+
+# Performance/Load testing: Run test multiple times in parallel
+pytest -n 10 --count 50 -m excel_generation  # Run 50 times with 10 workers
+pytest -n 20 --count 100 -m smoke --reruns 2  # Heavy load with retries
+```
+
+**Notes:**
+- `--reruns 2` uses pytest-rerunfailures to improve resiliency in flaky environments
+- `--count N` uses pytest-repeat to run each test N times (useful for performance/load testing)
+
+#### Test Timeout Configuration
+
+Tests have a configurable timeout to prevent hanging. Default is **300 seconds (5 minutes)** per test.
+
+**Configure in .env file:**
+```properties
+TEST_TIMEOUT=600  # 10 minutes
+```
+
+**Override via pytest CLI:**
+```shell
+# Set timeout for this run
+pytest -n 10 -m smoke --timeout 900  # 15 minutes
+
+# Disable timeout (debugging)
+pytest -m slow_tests --timeout 0
+
+# Use default from .env or pytest.ini
+pytest -n 10 -m api  # Uses TEST_TIMEOUT from .env or 300s default
+```
+
+**Timeout Behavior:**
+
+When a test exceeds the configured timeout:
+- Test execution is **terminated immediately**
+- Test is marked as **FAILED** with a timeout error message
+- Stack trace shows where the test was when timeout occurred
+- Remaining tests continue execution normally
+
+**Example timeout failure:**
+```
+================================== FAILURES ===================================
+_________________ test_slow_workflow_execution ________________
+
+E   Failed: Timeout >300.0s
+
+tests/workflow/test_workflows.py:145: Failed
 ```
 
-Note: "--reruns 2" uses pytest-rerunfailures to improve resiliency in flaky environments.
+**Best Practices:**
+- Keep default timeout reasonable (5-10 minutes for E2E tests)
+- Increase timeout for specific slow tests using `@pytest.mark.timeout(900)`
+- Use timeout=0 only for debugging hanging tests
+- Consider if a test legitimately needs > 5 minutes (optimize if possible)
 
 ### UI tests (Playwright)
 

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/README.md

@@ -69,6 +69,7 @@ Optional defaults for pytest:
 codemie-test-harness config set PYTEST_MARKS "smoke"
 codemie-test-harness config set PYTEST_N 8
 codemie-test-harness config set PYTEST_RERUNS 2
+codemie-test-harness config set PYTEST_COUNT 10  # For performance testing (optional)
 ```
 
 #### Integration Categories & Management
@@ -266,6 +267,80 @@ codemie-test-harness --git-env github run --marks git
 codemie-test-harness run --marks ui --headless
 ```
 
+#### Performance and Load Testing
+
+Run tests multiple times in parallel to simulate load and measure performance:
+
+```shell
+# Performance test: Run test 50 times with 10 parallel workers
+codemie-test-harness run --marks excel_generation --count 50 -n 10
+
+# Heavy load test: 100 iterations with 20 workers
+codemie-test-harness run --marks excel_generation --count 100 -n 20 -v
+
+# Light load test with retries for stability
+codemie-test-harness run --marks smoke --count 25 -n 5 --reruns 2
+
+# Set default count in config for repeated use
+codemie-test-harness config set PYTEST_COUNT 30
+codemie-test-harness run --marks excel_generation -n 10  # Uses count=30 from config
+
+# Override config default for a specific run
+codemie-test-harness run --marks excel_generation --count 100 -n 20  # Overrides config
+```
+
+**Note:** The `--count` parameter requires the `pytest-repeat` plugin, which is included in the dependencies.
+
+#### Test Timeout Control
+
+Control per-test timeout to prevent tests from running indefinitely. Tests exceeding the timeout will be **automatically terminated and marked as FAILED**.
+
+**Configuration Priority**: CLI args → Environment variable → Config file → Default (300 seconds)
+
+**Usage Examples:**
+
+```shell
+# Via CLI argument (600 seconds = 10 minutes)
+codemie-test-harness run --timeout 600 --marks smoke
+
+# Via config file (persistent)
+codemie-test-harness config set TEST_TIMEOUT 900
+codemie-test-harness run --marks api
+
+# Via environment variable
+export TEST_TIMEOUT=300
+codemie-test-harness run --marks smoke
+
+# Disable timeout for debugging (use 0)
+codemie-test-harness run --timeout 0 --marks smoke
+
+# Override config default for specific run
+codemie-test-harness config set TEST_TIMEOUT 600
+codemie-test-harness run --timeout 1200 --marks slow  # Uses 1200, not 600
+```
+
+**Default**: 300 seconds (5 minutes) per test
+
+**What Happens on Timeout?**
+
+When a test exceeds the configured timeout:
+1. ✅ **Test is automatically terminated** - Execution stops immediately
+2. ✅ **Marked as FAILED** - Test result shows as failed with clear timeout message
+3. ✅ **Error details displayed** - Shows which test timed out and the configured limit
+4. ✅ **Remaining tests continue** - Other tests proceed normally
+5. ✅ **Stack trace captured** - Shows where the test was when timeout occurred
+
+**Example timeout error output:**
+```
+FAILED tests/test_slow_operation.py::test_data_processing - Failed: Timeout >300.0s
+```
+
+**Notes:**
+- Timeout applies to **individual test functions**, not the entire test run
+- Useful for preventing hanging tests in CI/CD pipelines
+- Consider increasing timeout for legitimate long-running operations (data processing, large file operations)
+- Timeout of 0 disables the timeout (use for debugging only)
+
 #### Assistant Chat Interface
 
 Interact directly with CodeMie assistants through the CLI:
@@ -341,6 +416,9 @@ codemie-test-harness config vars data-management
 
 # Test execution with multiple overrides
 codemie-test-harness run --marks "smoke and not ui" -n 10 --reruns 3 --headless
+
+# Performance testing with count parameter
+codemie-test-harness run --marks excel_generation --count 50 -n 10
 ```
 
 #### Common Test Markers
@@ -481,9 +559,60 @@ pytest -m not_for_parallel_run --reruns 2
 # API tests
 pytest -n 10 -m "api and not not_for_parallel_run" --reruns 2
 pytest -m not_for_parallel_run --reruns 3
+
+# Performance/Load testing: Run test multiple times in parallel
+pytest -n 10 --count 50 -m excel_generation  # Run 50 times with 10 workers
+pytest -n 20 --count 100 -m smoke --reruns 2  # Heavy load with retries
+```
+
+**Notes:**
+- `--reruns 2` uses pytest-rerunfailures to improve resiliency in flaky environments
+- `--count N` uses pytest-repeat to run each test N times (useful for performance/load testing)
+
+#### Test Timeout Configuration
+
+Tests have a configurable timeout to prevent hanging. Default is **300 seconds (5 minutes)** per test.
+
+**Configure in .env file:**
+```properties
+TEST_TIMEOUT=600  # 10 minutes
+```
+
+**Override via pytest CLI:**
+```shell
+# Set timeout for this run
+pytest -n 10 -m smoke --timeout 900  # 15 minutes
+
+# Disable timeout (debugging)
+pytest -m slow_tests --timeout 0
+
+# Use default from .env or pytest.ini
+pytest -n 10 -m api  # Uses TEST_TIMEOUT from .env or 300s default
+```
+
+**Timeout Behavior:**
+
+When a test exceeds the configured timeout:
+- Test execution is **terminated immediately**
+- Test is marked as **FAILED** with a timeout error message
+- Stack trace shows where the test was when timeout occurred
+- Remaining tests continue execution normally
+
+**Example timeout failure:**
+```
+================================== FAILURES ===================================
+_________________ test_slow_workflow_execution ________________
+
+E   Failed: Timeout >300.0s
+
+tests/workflow/test_workflows.py:145: Failed
 ```
 
-Note: "--reruns 2" uses pytest-rerunfailures to improve resiliency in flaky environments.
+**Best Practices:**
+- Keep default timeout reasonable (5-10 minutes for E2E tests)
+- Increase timeout for specific slow tests using `@pytest.mark.timeout(900)`
+- Use timeout=0 only for debugging hanging tests
+- Consider if a test legitimately needs > 5 minutes (optimize if possible)
 
 ### UI tests (Playwright)
 

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/cli/cli.py

@@ -15,6 +15,8 @@ from .constants import (
     KEY_MARKS,
     KEY_XDIST_N,
     KEY_RERUNS,
+    KEY_COUNT,
+    KEY_TIMEOUT,
     KEY_AUTH_SERVER_URL,
     KEY_AUTH_CLIENT_ID,
     KEY_AUTH_CLIENT_SECRET,
@@ -22,12 +24,10 @@ from .constants import (
     KEY_CODEMIE_API_DOMAIN,
     KEY_AUTH_USERNAME,
     KEY_AUTH_PASSWORD,
-    # integrations
     DEFAULT_MARKS,
     DEFAULT_XDIST_N,
     DEFAULT_RERUNS,
-    AUTH_KEYS,
-    INTEGRATION_KEYS,
+    DEFAULT_TIMEOUT,
 )
 from .utils import get_config_value, ensure_env_from_config
 from .runner import run_pytest
@@ -50,6 +50,18 @@ from .commands.marks_cmd import marks_cmd
 @click.option(
     "--reruns", envvar=KEY_RERUNS, type=int, help="Number of reruns for flaky tests"
 )
+@click.option(
+    "--count",
+    envvar=KEY_COUNT,
+    type=int,
+    help="Number of times to repeat each test (requires pytest-repeat)",
+)
+@click.option(
+    "--timeout",
+    envvar=KEY_TIMEOUT,
+    type=int,
+    help="Per-test timeout in seconds (default: 300)",
+)
 @click.option("--auth-server-url", envvar=KEY_AUTH_SERVER_URL, help="Auth server url")
 @click.option("--auth-client-id", envvar=KEY_AUTH_CLIENT_ID, help="Auth client id")
 @click.option(
@@ -68,6 +80,8 @@ def cli(
     marks: Optional[str],
     workers: Optional[int],
     reruns: Optional[int],
+    count: Optional[int],
+    timeout: Optional[int],
     auth_server_url: Optional[str],
     auth_client_id: Optional[str],
     auth_client_secret: Optional[str],
@@ -99,6 +113,16 @@ def cli(
         if reruns is not None
         else int(get_config_value(KEY_RERUNS, str(DEFAULT_RERUNS)))
     )
+    resolved_count = (
+        count
+        if count is not None
+        else (int(get_config_value(KEY_COUNT)) if get_config_value(KEY_COUNT) else None)
+    )
+    resolved_timeout = (
+        timeout
+        if timeout is not None
+        else int(get_config_value(KEY_TIMEOUT, str(DEFAULT_TIMEOUT)))
+    )
 
     # Ensure env vars. CLI args override env/config.
     provided = {
@@ -115,15 +139,27 @@ def cli(
         if v is not None and v != "":
             os.environ[k] = str(v)
     # populate any missing values from saved config
-    ensure_env_from_config(AUTH_KEYS + INTEGRATION_KEYS)
+    ensure_env_from_config()
 
     ctx.obj.update(
-        dict(marks=resolved_marks, workers=resolved_workers, reruns=resolved_reruns)
+        dict(
+            marks=resolved_marks,
+            workers=resolved_workers,
+            reruns=resolved_reruns,
+            count=resolved_count,
+            timeout=resolved_timeout,
+        )
     )
 
     # default behavior
     if ctx.invoked_subcommand is None and not ctx.resilient_parsing:
-        run_pytest(resolved_workers, resolved_marks, resolved_reruns)
+        run_pytest(
+            resolved_workers,
+            resolved_marks,
+            resolved_reruns,
+            resolved_count,
+            resolved_timeout,
+        )
 
 
 # Register subcommands

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/cli/commands/run_cmd.py

@@ -10,6 +10,16 @@ from ..runner import run_pytest
     "-n", "workers", type=int, help="Override number of xdist workers for this run"
 )
 @click.option("--reruns", type=int, help="Override number of reruns for this run")
+@click.option(
+    "--count",
+    type=int,
+    help="Number of times to repeat each test (requires pytest-repeat)",
+)
+@click.option(
+    "--timeout",
+    type=int,
+    help="Per-test timeout in seconds (overrides config/default)",
+)
 @click.argument("extra", nargs=-1)
 @click.pass_context
 def run_cmd(
@@ -17,14 +27,27 @@ def run_cmd(
     marks: Optional[str],
     workers: Optional[int],
     reruns: Optional[int],
+    count: Optional[int],
+    timeout: Optional[int],
     extra: Tuple[str, ...],
 ):
     """Run pytest with configured options.
 
     Example: codemie-test-harness run --marks "smoke and not ui" -n 8 --reruns 2 -k keyword
+    Example with repeat: codemie-test-harness run --marks excel_generation --count 50 -n 10
+    Example with timeout: codemie-test-harness run --marks slow --timeout 600 -n 4
     """
     resolved_marks = marks or ctx.obj.get("marks")
     resolved_workers = workers if workers is not None else ctx.obj.get("workers")
     resolved_reruns = reruns if reruns is not None else ctx.obj.get("reruns")
+    resolved_count = count if count is not None else ctx.obj.get("count")
+    resolved_timeout = timeout if timeout is not None else ctx.obj.get("timeout")
 
-    run_pytest(int(resolved_workers), str(resolved_marks), int(resolved_reruns), extra)
+    run_pytest(
+        int(resolved_workers),
+        str(resolved_marks),
+        int(resolved_reruns),
+        resolved_count,
+        resolved_timeout,
+        extra,
+    )

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/cli/constants.py

@@ -19,6 +19,7 @@ KEY_CODEMIE_API_DOMAIN = "CODEMIE_API_DOMAIN"
 KEY_MARKS = "PYTEST_MARKS"
 KEY_XDIST_N = "PYTEST_N"
 KEY_RERUNS = "PYTEST_RERUNS"
+KEY_COUNT = "PYTEST_COUNT"
 
 # === COMPLETE INTEGRATION CREDENTIALS KEYS ===
 # Version Control Systems (GitLab, GitHub)

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/cli/runner.py

@@ -49,7 +49,12 @@ def resolve_tests_path_and_root() -> tuple[str, str]:
 
 
 def build_pytest_cmd(
-    workers: int, marks: str, reruns: int, extra: Iterable[str] | None = None
+    workers: int,
+    marks: str,
+    reruns: int,
+    count: int | None = None,
+    timeout: int | None = None,
+    extra: Iterable[str] | None = None,
 ) -> tuple[List[str], str]:
     tests_path, root_dir = resolve_tests_path_and_root()
     cmd = [sys.executable, "-m", "pytest", tests_path]
@@ -59,6 +64,10 @@ def build_pytest_cmd(
         cmd += ["-m", str(marks)]
     if reruns and int(reruns) > 0:
         cmd += ["--reruns", str(reruns)]
+    if count and int(count) > 0:
+        cmd += ["--count", str(count)]
+    if timeout and int(timeout) > 0:
+        cmd += ["--timeout", str(timeout)]
     if extra:
         cmd += list(extra)
     return cmd, root_dir
@@ -97,11 +106,16 @@ def validate_marks_expression(marks: str) -> None:
 
 
 def run_pytest(
-    workers: int, marks: str, reruns: int, extra: Iterable[str] | None = None
+    workers: int,
+    marks: str,
+    reruns: int,
+    count: int | None = None,
+    timeout: int | None = None,
+    extra: Iterable[str] | None = None,
 ) -> None:
     # Validate marks before running pytest
     validate_marks_expression(marks)
 
-    cmd, root_dir = build_pytest_cmd(workers, marks, reruns, extra)
+    cmd, root_dir = build_pytest_cmd(workers, marks, reruns, count, timeout, extra)
     CONSOLE.print(f"[cyan]Running:[/] {' '.join(cmd)} (cwd={root_dir})")
     raise SystemExit(subprocess.call(cmd, cwd=root_dir))

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/cli/utils.py

@@ -37,9 +37,19 @@ def set_config_value(key: str, value: Any) -> None:
     save_config(cfg)
 
 
-def ensure_env_from_config(keys: list[str]) -> None:
-    # Populate missing env vars from config file if present
+def ensure_env_from_config(keys: Optional[list[str]] = None) -> None:
+    """
+    Populate missing environment variables from config file.
+
+    Args:
+        keys: Optional list of specific keys to load. If None, loads all keys from config.
+    """
     cfg = load_config()
+
+    # If no specific keys provided, load all keys from config
+    if keys is None:
+        keys = list(cfg.keys())
+
     for k in keys:
         if k not in os.environ and k in cfg:
             os.environ[k] = str(cfg[k])

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/pytest.ini

@@ -1,5 +1,8 @@
 [pytest]
 addopts = -v
+timeout = 300
+timeout_method = signal
+timeout_func_only = true
 filterwarnings =
     ignore::pytest.PytestUnknownMarkWarning
     ignore::urllib3.exceptions.InsecureRequestWarning

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/tests/assistant/test_assistants.py

@@ -436,7 +436,9 @@ def test_excel_tool_extended_functionality(
     - Multi-sheet comprehensive analysis
 
     """
-    assistant_instance = assistant()
+    assistant_instance = assistant(
+        system_prompt="You have all required information in initial prompt. Do not ask additional questions and proceed with request."
+    )
 
     uploaded_file = assistant_utils.upload_file_to_chat(
         FILES_PATH / "test_extended.xlsx"
@@ -490,3 +492,17 @@ def test_docx_tool_extended_functionality(
 
     assert_tool_triggered(Default.DOCX_TOOL, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
+
+
+@pytest.mark.assistant
+# @pytest.mark.api
+@pytest.mark.smoke
+@pytest.mark.excel_generation
+def test_excel_file_generation(assistant_utils, assistant, similarity_check):
+    assistant_instance = assistant()
+
+    response = assistant_utils.ask_assistant(
+        assistant_instance, "Generate excel with 5 sheets with random data"
+    )
+
+    similarity_check.check_similarity(response, "expected_response_here")

{codemie_test_harness-0.1.183 → codemie_test_harness-0.1.198}/codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py

@@ -69,5 +69,5 @@ def test_assistant_with_sonar_tools(
         assistant, prompt, minimal_response=False
     )
 
-    assert_tool_triggered(tool_name, triggered_tools)
+    assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
     similarity_check.check_similarity(response, expected)