codemie-test-harness 0.1.197__py3-none-any.whl → 0.1.198__py3-none-any.whl

codemie_test_harness/cli/cli.py

@@ -16,6 +16,7 @@ from .constants import (
     KEY_XDIST_N,
     KEY_RERUNS,
     KEY_COUNT,
+    KEY_TIMEOUT,
     KEY_AUTH_SERVER_URL,
     KEY_AUTH_CLIENT_ID,
     KEY_AUTH_CLIENT_SECRET,
@@ -26,6 +27,7 @@ from .constants import (
     DEFAULT_MARKS,
     DEFAULT_XDIST_N,
     DEFAULT_RERUNS,
+    DEFAULT_TIMEOUT,
 )
 from .utils import get_config_value, ensure_env_from_config
 from .runner import run_pytest
@@ -54,6 +56,12 @@ from .commands.marks_cmd import marks_cmd
     type=int,
     help="Number of times to repeat each test (requires pytest-repeat)",
 )
+@click.option(
+    "--timeout",
+    envvar=KEY_TIMEOUT,
+    type=int,
+    help="Per-test timeout in seconds (default: 300)",
+)
 @click.option("--auth-server-url", envvar=KEY_AUTH_SERVER_URL, help="Auth server url")
 @click.option("--auth-client-id", envvar=KEY_AUTH_CLIENT_ID, help="Auth client id")
 @click.option(
@@ -73,6 +81,7 @@ def cli(
     workers: Optional[int],
     reruns: Optional[int],
     count: Optional[int],
+    timeout: Optional[int],
     auth_server_url: Optional[str],
     auth_client_id: Optional[str],
     auth_client_secret: Optional[str],
@@ -109,6 +118,11 @@ def cli(
         if count is not None
         else (int(get_config_value(KEY_COUNT)) if get_config_value(KEY_COUNT) else None)
     )
+    resolved_timeout = (
+        timeout
+        if timeout is not None
+        else int(get_config_value(KEY_TIMEOUT, str(DEFAULT_TIMEOUT)))
+    )
 
     # Ensure env vars. CLI args override env/config.
     provided = {
@@ -133,12 +147,19 @@ def cli(
             workers=resolved_workers,
             reruns=resolved_reruns,
             count=resolved_count,
+            timeout=resolved_timeout,
         )
     )
 
     # default behavior
     if ctx.invoked_subcommand is None and not ctx.resilient_parsing:
-        run_pytest(resolved_workers, resolved_marks, resolved_reruns, resolved_count)
+        run_pytest(
+            resolved_workers,
+            resolved_marks,
+            resolved_reruns,
+            resolved_count,
+            resolved_timeout,
+        )
 
 
 # Register subcommands

codemie_test_harness/cli/commands/config_cmd.py

@@ -5,9 +5,9 @@ from typing import Dict, List
 from ..constants import (
     CONSOLE,
     CREDENTIAL_CATEGORIES,
-    INTEGRATION_KEYS,
     mask_sensitive_value,
     is_sensitive_key,
+    INTEGRATION_KEYS,
 )
 from ..utils import (
     load_config,

codemie_test_harness/cli/commands/run_cmd.py

@@ -15,6 +15,11 @@ from ..runner import run_pytest
     type=int,
     help="Number of times to repeat each test (requires pytest-repeat)",
 )
+@click.option(
+    "--timeout",
+    type=int,
+    help="Per-test timeout in seconds (overrides config/default)",
+)
 @click.argument("extra", nargs=-1)
 @click.pass_context
 def run_cmd(
@@ -23,22 +28,26 @@ def run_cmd(
     workers: Optional[int],
     reruns: Optional[int],
     count: Optional[int],
+    timeout: Optional[int],
     extra: Tuple[str, ...],
 ):
     """Run pytest with configured options.
 
     Example: codemie-test-harness run --marks "smoke and not ui" -n 8 --reruns 2 -k keyword
     Example with repeat: codemie-test-harness run --marks excel_generation --count 50 -n 10
+    Example with timeout: codemie-test-harness run --marks slow --timeout 600 -n 4
     """
     resolved_marks = marks or ctx.obj.get("marks")
     resolved_workers = workers if workers is not None else ctx.obj.get("workers")
     resolved_reruns = reruns if reruns is not None else ctx.obj.get("reruns")
     resolved_count = count if count is not None else ctx.obj.get("count")
+    resolved_timeout = timeout if timeout is not None else ctx.obj.get("timeout")
 
     run_pytest(
         int(resolved_workers),
         str(resolved_marks),
         int(resolved_reruns),
         resolved_count,
+        resolved_timeout,
         extra,
     )

codemie_test_harness/cli/constants.py

@@ -147,6 +147,16 @@ DEFAULT_MARKS = "smoke"
 DEFAULT_XDIST_N = 8
 DEFAULT_RERUNS = 2
 
+AUTH_KEYS = [
+    KEY_AUTH_SERVER_URL,
+    KEY_AUTH_CLIENT_ID,
+    KEY_AUTH_CLIENT_SECRET,
+    KEY_AUTH_USERNAME,
+    KEY_AUTH_PASSWORD,
+    KEY_AUTH_REALM_NAME,
+    KEY_CODEMIE_API_DOMAIN,
+]
+
 # === CREDENTIAL CATEGORIES ===
 VERSION_CONTROL_KEYS = [
     KEY_GIT_ENV,
@@ -274,6 +284,17 @@ DATA_MANAGEMENT_KEYS = [
     KEY_MSSQL_PASSWORD,
 ]
 
+# Combined lists for backwards compatibility
+INTEGRATION_KEYS = (
+    VERSION_CONTROL_KEYS
+    + PROJECT_MANAGEMENT_KEYS
+    + CLOUD_PROVIDERS_KEYS
+    + DEVELOPMENT_TOOLS_KEYS
+    + NOTIFICATIONS_KEYS
+    + RESEARCH_TOOLS_KEYS
+    + DATA_MANAGEMENT_KEYS
+)
+
 # Category mapping for CLI
 CREDENTIAL_CATEGORIES = {
     "version-control": {

codemie_test_harness/cli/runner.py

@@ -53,6 +53,7 @@ def build_pytest_cmd(
     marks: str,
     reruns: int,
     count: int | None = None,
+    timeout: int | None = None,
     extra: Iterable[str] | None = None,
 ) -> tuple[List[str], str]:
     tests_path, root_dir = resolve_tests_path_and_root()
@@ -65,6 +66,8 @@ def build_pytest_cmd(
         cmd += ["--reruns", str(reruns)]
     if count and int(count) > 0:
         cmd += ["--count", str(count)]
+    if timeout and int(timeout) > 0:
+        cmd += ["--timeout", str(timeout)]
     if extra:
         cmd += list(extra)
     return cmd, root_dir
@@ -107,11 +110,12 @@ def run_pytest(
     marks: str,
     reruns: int,
     count: int | None = None,
+    timeout: int | None = None,
     extra: Iterable[str] | None = None,
 ) -> None:
     # Validate marks before running pytest
     validate_marks_expression(marks)
 
-    cmd, root_dir = build_pytest_cmd(workers, marks, reruns, count, extra)
+    cmd, root_dir = build_pytest_cmd(workers, marks, reruns, count, timeout, extra)
     CONSOLE.print(f"[cyan]Running:[/] {' '.join(cmd)} (cwd={root_dir})")
     raise SystemExit(subprocess.call(cmd, cwd=root_dir))

codemie_test_harness/pytest.ini

@@ -1,5 +1,8 @@
 [pytest]
 addopts = -v
+timeout = 300
+timeout_method = signal
+timeout_func_only = true
 filterwarnings =
     ignore::pytest.PytestUnknownMarkWarning
     ignore::urllib3.exceptions.InsecureRequestWarning

codemie_test_harness/tests/conftest.py

@@ -25,6 +25,7 @@ from codemie_sdk.models.integration import (
 from codemie_sdk.models.workflow import WorkflowCreateRequest, WorkflowMode, Workflow
 
 from codemie_test_harness.tests import PROJECT, autotest_entity_prefix
+from codemie_test_harness.tests.test_data.file_test_data import file_test_data
 from codemie_test_harness.tests.test_data.google_datasource_test_data import (
     GOOGLE_DOC_URL,
 )
@@ -32,7 +33,7 @@ from codemie_test_harness.tests.utils.assistant_utils import AssistantUtils
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.base_utils import get_random_name, wait_for_entity
 from codemie_test_harness.tests.utils.client_factory import get_client
-from codemie_test_harness.tests.utils.constants import TESTS_PATH
+from codemie_test_harness.tests.utils.constants import TESTS_PATH, FILES_PATH
 from codemie_test_harness.tests.utils.conversation_utils import ConversationUtils
 from codemie_test_harness.tests.utils.datasource_utils import DataSourceUtils
 from codemie_test_harness.tests.utils.gitbud_utils import GitBudUtils
@@ -458,6 +459,21 @@ def code_datasource(
         datasource_utils.delete_datasource(datasource)
 
 
+@pytest.fixture(scope="session")
+def file_datasource(datasource_utils, default_embedding_llm):
+    file_name = file_test_data[2][0]
+
+    datasource = datasource_utils.create_file_datasource(
+        name=get_random_name(),
+        description=f"[Autotest] {file_name} with {default_embedding_llm.base_name} embedding model",
+        files=[str(FILES_PATH / file_name)],
+        embeddings_model=default_embedding_llm.base_name,
+    )
+    yield datasource
+    if datasource:
+        datasource_utils.delete_datasource(datasource)
+
+
 @pytest.fixture(scope="session")
 def gitlab_datasource(datasource_utils, gitlab_integration, default_embedding_llm):
     datasource = datasource_utils.create_gitlab_datasource(

codemie_test_harness/tests/integrations/user/test_default_integrations.py

@@ -37,6 +37,7 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
 from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
     rp_test_data,
 )
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.constants import test_project_name
 
@@ -80,7 +81,11 @@ def test_assistant_with_default_integration_cloud(
     # create an assistant
     cloud_assistant = assistant(toolkit, tool_name, project_name=test_project_name)
 
-    response = assistant_utils.ask_assistant(cloud_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        cloud_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -119,7 +124,11 @@ def test_assistant_with_default_integration_ado(
     # create an assistant
     ado_assistant = assistant(toolkit, tool_name, project_name=test_project_name)
 
-    response = assistant_utils.ask_assistant(ado_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -163,7 +172,11 @@ def test_assistant_with_default_integration_codebase(
         toolkit, CodeBaseTool.SONAR, project_name=test_project_name
     )
 
-    response = assistant_utils.ask_assistant(sonar_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        sonar_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -216,7 +229,11 @@ def test_assistant_with_default_integration_git(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -255,7 +272,11 @@ def test_assistant_with_default_integration_jira(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
@@ -292,7 +313,11 @@ def test_assistant_with_default_integration_email(
         Toolkit.NOTIFICATION, NotificationTool.EMAIL, project_name=test_project_name
     )
 
-    response = assistant_utils.ask_assistant(email_assistant, EMAIL_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        email_assistant, EMAIL_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(NotificationTool.EMAIL, triggered_tools)
 
     similarity_check.check_similarity(response, EMAIL_RESPONSE)
 
@@ -329,7 +354,11 @@ def test_assistant_with_default_integration_keycloak(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        keycloak_assistant, KEYCLOAK_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
 
     similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
 
@@ -369,6 +398,10 @@ def test_assistant_with_default_integration_report_portal(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(report_portal_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        report_portal_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(ReportPortalTool.GET_DASHBOARD_DATA, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/llm/assistants/test_llm.py

@@ -3,11 +3,8 @@ from codemie_sdk.models.assistant import ToolKitDetails, ToolDetails
 from hamcrest import assert_that, has_item
 from codemie_test_harness.tests.enums.model_types import ModelTypes
 from codemie_test_harness.tests.enums.tools import Toolkit, FileManagementTool
-from codemie_test_harness.tests.test_data.file_test_data import file_test_data
 from codemie_test_harness.tests.test_data.llm_test_data import MODEL_RESPONSES
-from codemie_test_harness.tests.utils.base_utils import get_random_name
 from codemie_test_harness.tests.utils.client_factory import get_client
-from codemie_test_harness.tests.utils.constants import FILES_PATH
 from codemie_test_harness.tests.utils.env_resolver import get_environment
 from codemie_test_harness.tests.utils.pytest_utils import check_mark
 
@@ -121,6 +118,7 @@ def test_assistant_with_different_models_with_datasource_attached(
     datasource_utils,
     default_embedding_llm,
     kb_context,
+    file_datasource,
 ):
     assert_that(
         [row.base_name for row in llm_utils.list_llm_models()],
@@ -128,17 +126,8 @@ def test_assistant_with_different_models_with_datasource_attached(
         f"{model_type} is missing in backend response",
     )
 
-    file_name = file_test_data[2][0]
-
-    datasource = datasource_utils.create_file_datasource(
-        name=get_random_name(),
-        description=f"[Autotest] {file_name} with {default_embedding_llm.base_name} embedding model",
-        files=[str(FILES_PATH / file_name)],
-        embeddings_model=default_embedding_llm.base_name,
-    )
-
     assistant = assistant_utils.create_assistant(
-        model_type, context=[kb_context(datasource)]
+        model_type, context=[kb_context(file_datasource)]
     )
     response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")