codemie-test-harness 0.1.179__py3-none-any.whl → 0.1.181__py3-none-any.whl

codemie_test_harness/cli/cli.py

@@ -35,6 +35,7 @@ from .commands.config_cmd import config_cmd
 from .commands.run_cmd import run_cmd
 from .commands.assistant_cmd import assistant_cmd
 from .commands.workflow_cmd import workflow_cmd
+from .commands.marks_cmd import marks_cmd
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)
@@ -130,6 +131,7 @@ cli.add_command(config_cmd)
 cli.add_command(run_cmd)
 cli.add_command(assistant_cmd)
 cli.add_command(workflow_cmd)
+cli.add_command(marks_cmd)
 
 
 if __name__ == "__main__":  # pragma: no cover

codemie_test_harness/cli/commands/marks_cmd.py

@@ -0,0 +1,39 @@
+"""CLI command for listing pytest marks."""
+
+from __future__ import annotations
+import click
+from ..marks_utils import discover_all_marks, print_marks_list
+from ..constants import CONSOLE
+
+
+@click.command(name="marks")
+@click.option(
+    "--verbose",
+    "-v",
+    is_flag=True,
+    help="Show detailed information in table format with line numbers",
+)
+@click.option(
+    "--count", "-c", is_flag=True, help="Show only the count of available marks"
+)
+def marks_cmd(verbose: bool, count: bool):
+    """List all available pytest marks in the test suite.
+
+    Examples:
+        codemie-test-harness marks                    # List all marks
+        codemie-test-harness marks --verbose          # List marks with file details in table format
+        codemie-test-harness marks --count            # Show only count
+    """
+    try:
+        if count:
+            # For count mode, we don't need detailed information
+            marks_info = discover_all_marks(include_details=False)
+            CONSOLE.print(f"[green]{len(marks_info)}[/green] pytest marks available")
+            return
+
+        # Let print_marks_list handle the discovery with appropriate details
+        print_marks_list(marks_info=None, show_files=verbose)
+
+    except Exception as e:
+        CONSOLE.print(f"[red]Error discovering marks: {str(e)}[/red]")
+        raise click.ClickException("Failed to discover pytest marks")

codemie_test_harness/cli/marks_utils.py

@@ -0,0 +1,338 @@
+"""Utilities for handling pytest marks in the test harness."""
+
+import re
+import ast
+from pathlib import Path
+from typing import Set, List, Dict
+from .constants import CONSOLE
+from rich.table import Table
+
+try:
+    from importlib.resources import files as pkg_files
+except Exception:
+    pkg_files = None
+
+
+def extract_marks_from_file(
+    file_path: Path, include_lines: bool = False
+) -> Set[str] | Dict[str, List[int]]:
+    """Extract pytest marks from a Python test file.
+
+    Args:
+        file_path: Path to the Python file to analyze
+        include_lines: If True, return dict with mark names and line numbers
+
+    Returns:
+        Set of mark names found in the file, or dict mapping marks to line numbers
+    """
+    if include_lines:
+        marks_with_lines = {}
+    else:
+        marks = set()
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            content = f.read()
+
+        # Parse the file to AST for reliable extraction
+        try:
+            tree = ast.parse(content)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.FunctionDef):
+                    for decorator in node.decorator_list:
+                        mark_name = _extract_mark_from_decorator(decorator)
+                        if mark_name:
+                            if include_lines:
+                                if mark_name not in marks_with_lines:
+                                    marks_with_lines[mark_name] = []
+                                marks_with_lines[mark_name].append(decorator.lineno)
+                            else:
+                                marks.add(mark_name)
+        except SyntaxError:
+            # If AST parsing fails, fall back to regex
+            pass
+
+        # Additional regex-based extraction for edge cases
+        if include_lines:
+            regex_marks_with_lines = _extract_marks_with_regex_and_lines(content)
+            for mark, lines in regex_marks_with_lines.items():
+                if mark not in marks_with_lines:
+                    marks_with_lines[mark] = []
+                marks_with_lines[mark].extend(lines)
+        else:
+            regex_marks = _extract_marks_with_regex(content)
+            marks.update(regex_marks)
+
+    except (IOError, UnicodeDecodeError):
+        # Skip files that can't be read
+        pass
+
+    return marks_with_lines if include_lines else marks
+
+
+def _extract_mark_from_decorator(decorator) -> str | None:
+    """Extract mark name from AST decorator node."""
+    mark_name = None
+
+    if isinstance(decorator, ast.Attribute):
+        # @pytest.mark.some_mark
+        if (
+            isinstance(decorator.value, ast.Attribute)
+            and isinstance(decorator.value.value, ast.Name)
+            and decorator.value.value.id == "pytest"
+            and decorator.value.attr == "mark"
+        ):
+            mark_name = decorator.attr
+    elif isinstance(decorator, ast.Call):
+        # @pytest.mark.some_mark(...) with arguments
+        if (
+            isinstance(decorator.func, ast.Attribute)
+            and isinstance(decorator.func.value, ast.Attribute)
+            and isinstance(decorator.func.value.value, ast.Name)
+            and decorator.func.value.value.id == "pytest"
+            and decorator.func.value.attr == "mark"
+        ):
+            mark_name = decorator.func.attr
+
+    # Filter out pytest built-in decorators that aren't marks
+    if mark_name in {
+        "parametrize",
+        "fixture",
+        "skip",
+        "skipif",
+        "xfail",
+        "usefixtures",
+        "testcase",
+        "todo",
+    }:
+        return None
+
+    return mark_name
+
+
+def _extract_marks_with_regex(content: str) -> Set[str]:
+    """Extract marks using regex as fallback."""
+    marks = set()
+
+    # Pattern to match @pytest.mark.mark_name
+    pattern = r"@pytest\.mark\.(\w+)"
+    matches = re.findall(pattern, content)
+
+    for match in matches:
+        # Filter out pytest built-in decorators that aren't marks
+        if match not in {
+            "parametrize",
+            "fixture",
+            "skip",
+            "skipif",
+            "xfail",
+            "usefixtures",
+            "todo",
+        }:
+            marks.add(match)
+
+    return marks
+
+
+def _extract_marks_with_regex_and_lines(content: str) -> Dict[str, List[int]]:
+    """Extract marks with line numbers using regex as fallback."""
+    marks_with_lines = {}
+    lines = content.split("\n")
+
+    # Pattern to match @pytest.mark.mark_name
+    pattern = r"@pytest\.mark\.(\w+)"
+
+    for line_num, line in enumerate(lines, 1):
+        matches = re.findall(pattern, line)
+        for match in matches:
+            # Filter out pytest built-in decorators that aren't marks
+            if match not in {
+                "parametrize",
+                "fixture",
+                "skip",
+                "skipif",
+                "xfail",
+                "usefixtures",
+                "todo",
+            }:
+                if match not in marks_with_lines:
+                    marks_with_lines[match] = []
+                marks_with_lines[match].append(line_num)
+
+    return marks_with_lines
+
+
+def _resolve_tests_path() -> str:
+    """Resolve tests path for mark discovery.
+
+    Returns:
+        Path to tests directory
+    """
+    # 1) Use importlib.resources to locate installed package "tests"
+    if pkg_files is not None:
+        try:
+            # Try to locate codemie_test_harness.tests package
+            tests_dir = Path(str(pkg_files("codemie_test_harness.tests")))
+            return str(tests_dir)
+        except Exception:
+            pass
+
+    # 2) Fallback to repo layout when running from source
+    # marks_utils.py -> cli -> codemie_test_harness -> <repo_root>
+    codemie_test_harness_root = (
+        Path(__file__).resolve().parents[1]
+    )  # codemie_test_harness directory
+    tests_path = str(codemie_test_harness_root / "tests")
+    return tests_path
+
+
+def discover_all_marks(
+    include_details: bool = False,
+) -> Dict[str, List[str]] | Dict[str, List[Dict]]:
+    """Discover all pytest marks used in the test suite.
+
+    Args:
+        include_details: If True, return detailed info with line numbers
+
+    Returns:
+        Dictionary with mark names as keys and list of files (or detailed info) as values
+    """
+    tests_path = _resolve_tests_path()
+    tests_dir = Path(tests_path)
+
+    if not tests_dir.exists():
+        return {}
+
+    marks_info = {}
+
+    # Find all Python test files
+    for py_file in tests_dir.rglob("*.py"):
+        if py_file.name.startswith("test_") or py_file.name.endswith("_test.py"):
+            rel_path = str(py_file.relative_to(tests_dir))
+
+            if include_details:
+                marks_with_lines = extract_marks_from_file(py_file, include_lines=True)
+
+                for mark, lines in marks_with_lines.items():
+                    if mark not in marks_info:
+                        marks_info[mark] = []
+
+                    for line in lines:
+                        marks_info[mark].append({"file": rel_path, "line": line})
+            else:
+                marks = extract_marks_from_file(py_file)
+
+                for mark in marks:
+                    if mark not in marks_info:
+                        marks_info[mark] = []
+                    # Store relative path for readability
+                    if rel_path not in marks_info[mark]:
+                        marks_info[mark].append(rel_path)
+
+    return marks_info
+
+
+def get_all_available_marks() -> List[str]:
+    """Get sorted list of all available pytest marks."""
+    marks_to_files = discover_all_marks()
+    return sorted(marks_to_files.keys())
+
+
+def is_valid_mark_expression(
+    expression: str, available_marks: List[str]
+) -> tuple[bool, List[str]]:
+    """Validate if a mark expression contains only known marks.
+
+    Args:
+        expression: The pytest mark expression (e.g., "smoke and ui", "not integration")
+        available_marks: List of all available marks
+
+    Returns:
+        Tuple of (is_valid, list_of_unknown_marks)
+    """
+    if not expression or not expression.strip():
+        return True, []
+
+    # Extract mark names from the expression
+    # This regex finds word tokens that could be mark names
+    potential_marks = re.findall(r"\b(\w+)\b", expression)
+
+    # Filter out logical operators and known keywords
+    logical_keywords = {"and", "or", "not", "true", "false"}
+    unknown_marks = []
+
+    available_marks_set = set(available_marks)
+
+    for mark in potential_marks:
+        if mark not in logical_keywords and mark not in available_marks_set:
+            unknown_marks.append(mark)
+
+    return len(unknown_marks) == 0, unknown_marks
+
+
+def print_marks_list(marks_info=None, show_files: bool = False):
+    """Print formatted list of available marks.
+
+    Args:
+        marks_info: Optional pre-computed marks dictionary
+        show_files: Whether to show detailed information in table format
+    """
+    if marks_info is None:
+        marks_info = discover_all_marks(include_details=show_files)
+
+    if not marks_info:
+        CONSOLE.print("[yellow]No pytest marks found in test files.[/yellow]")
+        return
+
+    if show_files:
+        # Verbose mode - show table with detailed information
+        CONSOLE.print(
+            f"\n[bold cyan]Available pytest marks ({len(marks_info)} total):[/bold cyan]\n"
+        )
+
+        table = Table(show_header=True, header_style="bold magenta")
+        table.add_column("Mark", style="bold green", width=25)
+        table.add_column("File", style="cyan", width=110)
+        table.add_column("Usage Count", style="blue", justify="right", width=12)
+
+        for mark in sorted(marks_info.keys()):
+            details = marks_info[mark]
+
+            # Group by file and count occurrences
+            file_counts = {}
+            for detail in details:
+                file_path = detail["file"]
+                if file_path not in file_counts:
+                    file_counts[file_path] = []
+                file_counts[file_path].append(detail["line"])
+
+            # Add rows for each file
+            first_row = True
+            for file_path, lines in sorted(file_counts.items()):
+                mark_display = (
+                    mark if first_row else ""
+                )  # Just the mark name, no @pytest.mark. prefix
+                usage_count = str(len(lines))
+
+                table.add_row(mark_display, file_path, usage_count)
+                first_row = False
+
+            # Add separator row if not the last mark
+            if mark != sorted(marks_info.keys())[-1]:
+                table.add_row("", "", "", style="dim")
+
+        CONSOLE.print(table)
+        CONSOLE.print(f"\n[dim]Total: {len(marks_info)} unique marks found[/dim]")
+    else:
+        # Simple mode - just list marks
+        CONSOLE.print(
+            f"\n[bold cyan]Available pytest marks ({len(marks_info)} total):[/bold cyan]\n"
+        )
+
+        for mark in sorted(marks_info.keys()):
+            files = marks_info[mark]
+            CONSOLE.print(f"[green]• {mark}[/green] [dim]({len(files)} files)[/dim]")
+
+        CONSOLE.print(
+            "\n[dim]Use --verbose to see detailed information in table format.[/dim]"
+        )

codemie_test_harness/cli/runner.py

@@ -10,6 +10,11 @@ except Exception:
     pkg_files = None
 
 from .constants import CONSOLE
+from .marks_utils import (
+    get_all_available_marks,
+    is_valid_mark_expression,
+    print_marks_list,
+)
 
 
 def resolve_tests_path_and_root() -> tuple[str, str]:
@@ -59,9 +64,44 @@ def build_pytest_cmd(
     return cmd, root_dir
 
 
+def validate_marks_expression(marks: str) -> None:
+    """Validate that the marks expression contains only known marks.
+
+    Args:
+        marks: The pytest marks expression to validate
+
+    Raises:
+        SystemExit: If unknown marks are found
+    """
+    if not marks or not marks.strip():
+        return
+
+    try:
+        available_marks = get_all_available_marks()
+        is_valid, unknown_marks = is_valid_mark_expression(marks, available_marks)
+
+        if not is_valid:
+            CONSOLE.print(
+                f"[red]Error: Unknown pytest mark(s) found: {', '.join(unknown_marks)}[/red]"
+            )
+            CONSOLE.print(f"\n[yellow]Expression used:[/yellow] {marks}")
+            print_marks_list(show_files=False)
+            CONSOLE.print(
+                "\n[dim]Use 'codemie-test-harness marks' to see all available marks[/dim]"
+            )
+            raise SystemExit(1)
+
+    except Exception as e:
+        # If mark validation fails for any reason, issue a warning but continue
+        CONSOLE.print(f"[yellow]Warning: Could not validate marks: {str(e)}[/yellow]")
+
+
 def run_pytest(
     workers: int, marks: str, reruns: int, extra: Iterable[str] | None = None
 ) -> None:
+    # Validate marks before running pytest
+    validate_marks_expression(marks)
+
     cmd, root_dir = build_pytest_cmd(workers, marks, reruns, extra)
     CONSOLE.print(f"[cyan]Running:[/] {' '.join(cmd)} (cwd={root_dir})")
     raise SystemExit(subprocess.call(cmd, cwd=root_dir))

codemie_test_harness/tests/assistant/datasource/test_code_datasource.py

@@ -4,13 +4,18 @@ import pytest
 from hamcrest import assert_that, equal_to
 from requests import HTTPError
 
+from codemie_test_harness.tests.enums.tools import Default
 from codemie_test_harness.tests.test_data.index_test_data import index_test_data
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.datasource
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "embedding_model",
     index_test_data,
@@ -35,22 +40,30 @@ def test_create_index_application_with_embedding_model(
         setting_id=git_integration.id, embeddings_model=embedding_model
     )
 
-    assistant = assistant(context=code_context(datasource))
+    assistant = assistant(
+        context=code_context(datasource), system_prompt="Run tools on each user prompt"
+    )
 
-    answer = assistant_utils.ask_assistant(assistant, question)
+    answer, triggered_tools = assistant_utils.ask_assistant(
+        assistant, question, minimal_response=False
+    )
+    assert_tool_triggered(Default.GET_REPOSITORY_FILE_TREE, triggered_tools)
     similarity_check.check_similarity(answer, expected_answer)
 
     datasource_utils.update_code_datasource(
         datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    answer = assistant_utils.ask_assistant(assistant, question)
+    answer, triggered_tools = assistant_utils.ask_assistant(
+        assistant, question, minimal_response=False
+    )
+    assert_tool_triggered(Default.GET_REPOSITORY_FILE_TREE, triggered_tools)
     similarity_check.check_similarity(answer, expected_answer)
 
 
 @pytest.mark.datasource
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 def test_edit_description_for_code_data_source(
     client,
     gitlab_datasource,
@@ -68,7 +81,7 @@ def test_edit_description_for_code_data_source(
 
 @pytest.mark.datasource
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 def test_create_code_datasource_with_existing_name(gitlab_datasource, datasource_utils):
     datasource = datasource_utils.get_datasource(gitlab_datasource.id)
 

codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py

@@ -13,7 +13,12 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 
 
 @pytest.fixture(scope="function")
@@ -29,7 +34,7 @@ def confluence_cloud_datasource(datasource_utils, confluence_cloud_integration):
 
 @pytest.mark.datasource
 @pytest.mark.project_management
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "datasource_fixture, prompt, expected_response",
     [
@@ -61,22 +66,30 @@ def test_create_datasource_with_confluence_and_confluence_cloud_integration(
     expected_response,
 ):
     datasource = request.getfixturevalue(datasource_fixture)
-    assistant = assistant(context=kb_context(datasource))
+    assistant = assistant(
+        context=kb_context(datasource), system_prompt="Run tools on each user prompt"
+    )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
     datasource_utils.update_confluence_datasource(
         datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.datasource
 @pytest.mark.confluence
-@pytest.mark.regression
+@pytest.mark.api
 def test_edit_description_for_confluence_data_source(
     client, confluence_datasource, datasource_utils
 ):
@@ -92,7 +105,7 @@ def test_edit_description_for_confluence_data_source(
 
 @pytest.mark.datasource
 @pytest.mark.confluence
-@pytest.mark.regression
+@pytest.mark.api
 def test_create_confluence_datasource_with_existing_name(
     datasource_utils, confluence_datasource
 ):

codemie_test_harness/tests/assistant/datasource/test_file_indexing.py

@@ -6,12 +6,17 @@ from hamcrest import (
 )
 from requests import HTTPError
 
+from codemie_test_harness.tests.enums.tools import Default
 from codemie_test_harness.tests.test_data.file_test_data import (
     file_test_data,
     large_files_test_data,
     RESPONSE_FOR_TWO_FILES_INDEXED,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.test_data.index_test_data import index_test_data
 from codemie_test_harness.tests.utils.client_factory import get_client
 from codemie_test_harness.tests.utils.constants import FILES_PATH
@@ -60,7 +65,7 @@ def pytest_generate_tests(metafunc):
 
 @pytest.mark.datasource
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_create_assistant_with_file_datasource(
     assistant,
@@ -82,7 +87,10 @@ def test_create_assistant_with_file_datasource(
     test_assistant = assistant(context=kb_context(datasource))
 
     prompt = "Show KB context. Return all information available in the context. Query may be 'Show content of the KB'"
-    response = assistant_utils.ask_assistant(test_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        test_assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
 
     similarity_check.check_similarity(
         response, expected_response, assistant_name=test_assistant.name
@@ -91,7 +99,7 @@ def test_create_assistant_with_file_datasource(
 
 @pytest.mark.datasource
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_edit_description_for_file_datasource(datasource_utils):
     initial_description = "[Autotest] Initial CSV datasource description"
@@ -115,7 +123,7 @@ def test_edit_description_for_file_datasource(datasource_utils):
 
 @pytest.mark.datasource
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 @pytest.mark.parametrize("file_name", large_files_test_data)
 def test_create_file_datasource_with_large_files(datasource_utils, file_name):
@@ -134,7 +142,7 @@ def test_create_file_datasource_with_large_files(datasource_utils, file_name):
 
 @pytest.mark.datasource
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_create_file_datasource_with_big_number_of_files(datasource_utils):
     files = [str(FILES_PATH / "test.txt") for _ in range(11)]
@@ -152,7 +160,7 @@ def test_create_file_datasource_with_big_number_of_files(datasource_utils):
 
 @pytest.mark.datasource
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_create_file_datasource_with_two_files(
     assistant, assistant_utils, datasource_utils, kb_context, similarity_check
@@ -168,8 +176,11 @@ def test_create_file_datasource_with_two_files(
 
     test_assistant = assistant(context=kb_context(datasource))
 
-    response = assistant_utils.ask_assistant(
-        test_assistant, "What types of data do we have available?"
+    response, triggered_tools = assistant_utils.ask_assistant(
+        test_assistant,
+        "What types of data do we have available?",
+        minimal_response=False,
     )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_TWO_FILES_INDEXED)