result-companion 0.0.1__py3-none-any.whl

result_companion/core/utils/logging_config.py

@@ -0,0 +1,115 @@
+import json
+import logging
+import os
+import tempfile
+from logging.handlers import RotatingFileHandler
+from typing import Dict
+
+from tqdm import tqdm
+
+
+class JsonFormatter(logging.Formatter):
+    """Formats log records as JSON."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Formats a log record as JSON."""
+        log_data = {
+            "timestamp": self.formatTime(record),
+            "logger": record.name,
+            "level": record.levelname,
+            "message": record.getMessage(),
+        }
+
+        if record.exc_info:
+            log_data["exception"] = self.formatException(record.exc_info)
+
+        return json.dumps(log_data)
+
+
+class TqdmLoggingHandler(logging.Handler):
+    """Logging handler that displays logs above tqdm progress bars."""
+
+    def __init__(self, level: int = logging.NOTSET):
+        super().__init__(level)
+        self.setFormatter(
+            logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        )
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            tqdm.write(self.format(record))
+            self.flush()
+        except Exception:
+            self.handleError(record)
+
+
+class LoggerRegistry:
+    """Registry for managing loggers with tqdm-compatible output."""
+
+    def __init__(self, default_log_level: int = logging.INFO):
+        self.loggers: Dict[str, logging.Logger] = {}
+        self.default_log_level: int = default_log_level
+        self._tqdm_handler = TqdmLoggingHandler(level=default_log_level)
+
+    def get_logger(self, name: str, use_tqdm: bool = True) -> logging.Logger:
+        """Get or create a logger by name."""
+        if name in self.loggers:
+            return self.loggers[name]
+
+        logger = _add_file_handler(name)
+
+        if use_tqdm and not any(
+            isinstance(h, TqdmLoggingHandler) for h in logger.handlers
+        ):
+            logger.addHandler(self._tqdm_handler)
+
+        self.loggers[name] = logger
+        return logger
+
+    def set_log_level(self, level: str | int) -> None:
+        """Set log level for console output only. File logging always captures DEBUG."""
+        if isinstance(level, str):
+            level = getattr(logging, level.upper(), logging.INFO)
+
+        self.default_log_level = level
+        self._tqdm_handler.setLevel(level)
+
+
+def _add_file_handler(name: str) -> logging.Logger:
+    """Adds JSON file handler to logger. Returns logger set to DEBUG level."""
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)
+
+    if logger.hasHandlers():
+        return logger
+
+    log_file_path = os.path.join(tempfile.gettempdir(), "result_companion.log")
+    try:
+        file_handler = RotatingFileHandler(
+            log_file_path, maxBytes=5 * 1024 * 1024, backupCount=3
+        )
+        file_handler.setLevel(logging.DEBUG)
+        file_handler.setFormatter(JsonFormatter())
+        logger.addHandler(file_handler)
+    except (OSError, IOError) as e:
+        logger.warning(f"Failed to write to log file {log_file_path}: {e}")
+
+    return logger
+
+
+# Module-level singleton and helpers
+logger_registry = LoggerRegistry()
+
+
+def set_global_log_level(log_level: str | int) -> None:
+    """Set log level for all loggers."""
+    logger_registry.set_log_level(log_level)
+
+
+def get_progress_logger(name: str = "RC") -> logging.Logger:
+    """Get a logger that works with progress bars."""
+    return logger_registry.get_logger(name)
+
+
+# Default logger
+logger = get_progress_logger("RC")

result_companion/core/utils/progress.py

@@ -0,0 +1,61 @@
+import asyncio
+from typing import Any, List, TypeVar
+
+from tqdm import tqdm
+
+T = TypeVar("T")
+
+
+async def run_tasks_with_progress(
+    coroutines: List,
+    semaphore: asyncio.Semaphore = None,
+    desc: str = "Processing tasks",
+) -> List[Any]:
+    """Run coroutines with progress bar tracking.
+
+    Args:
+        coroutines: List of coroutines to run.
+        semaphore: Optional semaphore to limit concurrency.
+        desc: Description to display in the progress bar.
+
+    Returns:
+        List of results from the coroutines.
+    """
+    if not coroutines:
+        return []
+
+    if semaphore is None:
+        semaphore = asyncio.Semaphore(1)
+
+    active_count = 0
+    lock = asyncio.Lock()
+
+    async def run_with_semaphore(coro):
+        nonlocal active_count
+        async with semaphore:
+            async with lock:
+                active_count += 1
+            try:
+                return await coro
+            finally:
+                async with lock:
+                    active_count -= 1
+
+    tasks = [asyncio.create_task(run_with_semaphore(coro)) for coro in coroutines]
+    results = [None] * len(tasks)
+    task_to_index = {task: i for i, task in enumerate(tasks)}
+    pending = set(tasks)
+
+    with tqdm(
+        total=len(tasks), desc=desc, position=0, leave=True, dynamic_ncols=True
+    ) as pbar:
+        while pending:
+            done, pending = await asyncio.wait(
+                pending, return_when=asyncio.FIRST_COMPLETED
+            )
+            for task in done:
+                results[task_to_index[task]] = task.result()
+                pbar.update(1)
+            pbar.set_description(f"{desc} ({active_count} active)")
+
+    return results

result_companion/entrypoints/cli/cli_app.py

@@ -0,0 +1,266 @@
+import subprocess
+from pathlib import Path
+from typing import List, Optional
+
+import typer
+from click import get_current_context
+
+from result_companion.core.analizers.local.ollama_install import (
+    auto_install_model,
+    auto_install_ollama,
+)
+from result_companion.core.analizers.local.ollama_runner import check_ollama_installed
+from result_companion.core.analizers.local.ollama_server_manager import (
+    OllamaServerManager,
+    resolve_server_manager,
+)
+from result_companion.core.utils.log_levels import LogLevels
+from result_companion.core.utils.logging_config import logger
+
+app = typer.Typer()
+setup_app = typer.Typer(help="Manage Ollama installation and models")
+app.add_typer(setup_app, name="setup")
+
+
+try:
+    from importlib.metadata import version as get_version
+except ImportError:
+    from importlib_metadata import version as get_version
+
+VERSION = get_version("result-companion")
+
+
+def version_callback(value: bool):
+    if value:
+        typer.echo(f"result-companion version: {VERSION}")
+        raise typer.Exit()
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    version: Optional[bool] = typer.Option(
+        None,
+        "--version",
+        "-v",
+        callback=version_callback,
+        is_eager=True,
+        help="Show the version and exit",
+    ),
+):
+    """
+    Result Companion CLI - Analyze Robot Framework results with LLM assistance.
+    """
+    # If no subcommand is provided, show help.
+    ctx = get_current_context()
+    if ctx.invoked_subcommand is None:
+        typer.echo(ctx.get_help())
+
+
+@app.command()
+def analyze(
+    output: Path = typer.Option(
+        ...,
+        "-o",
+        "--output",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+        help="Output.xml file path",
+    ),
+    log_level: LogLevels = typer.Option(
+        LogLevels.INFO,
+        "-l",
+        "--log-level",
+        help="Log level verbosity",
+        case_sensitive=True,
+    ),
+    config: Optional[Path] = typer.Option(
+        None,
+        "-c",
+        "--config",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+        help="YAML Config file path",
+    ),
+    report: Optional[str] = typer.Option(
+        None, "-r", "--report", help="Write LLM Report to HTML file"
+    ),
+    include_passing: bool = typer.Option(
+        False, "-i", "--include-passing", help="Include PASS test cases"
+    ),
+    include_tags: Optional[str] = typer.Option(
+        None,
+        "-I",
+        "--include",
+        help="Include tests by tags (comma-separated: 'smoke,critical*')",
+    ),
+    exclude_tags: Optional[str] = typer.Option(
+        None,
+        "-E",
+        "--exclude",
+        help="Exclude tests by tags (comma-separated: 'wip,bug*')",
+    ),
+    test_case_concurrency: Optional[int] = typer.Option(
+        None,
+        "--test-concurrency",
+        help="Test cases processed in parallel (overrides config)",
+    ),
+    chunk_concurrency: Optional[int] = typer.Option(
+        None,
+        "--chunk-concurrency",
+        help="Chunks per test case in parallel (overrides config)",
+    ),
+):
+    """Analyze Robot Framework test results with LLM assistance."""
+    typer.echo(f"Output: {output}")
+    typer.echo(f"Log Level: {log_level}")
+    typer.echo(f"Config: {config}")
+    typer.echo(f"Report: {report}")
+    typer.echo(f"Include Passing: {include_passing}")
+
+    # Parse CLI tag options
+    include_tag_list = (
+        [t.strip() for t in include_tags.split(",")] if include_tags else None
+    )
+    exclude_tag_list = (
+        [t.strip() for t in exclude_tags.split(",")] if exclude_tags else None
+    )
+
+    # Allow test injection via context, otherwise lazy import
+    ctx = get_current_context()
+    run = ctx.obj.get("analyze") if ctx.obj else None
+    if not run:
+        from result_companion.entrypoints.run_rc import run_rc
+
+        run = run_rc
+
+    run(
+        output,
+        log_level,
+        config,
+        report,
+        include_passing,
+        test_case_concurrency,
+        chunk_concurrency,
+        include_tag_list,
+        exclude_tag_list,
+    )
+
+
+# Setup commands using the original functions
+@setup_app.command("ollama")
+def setup_ollama(
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Force reinstallation even if already installed",
+    ),
+):
+    """Install Ollama on the local system."""
+    try:
+        typer.echo("Installing Ollama...")
+        auto_install_ollama()
+        typer.echo("Ollama installed successfully!")
+    except Exception as e:
+        typer.echo(f"Error during Ollama installation: {e}")
+        raise typer.Exit(code=1)
+
+
+def install_ollama_model(
+    model_name: str,
+    server_manager=OllamaServerManager,
+    installation_cmd: List[str] = ["ollama", "pull"],
+    ollama_list_cmd: List[str] = ["ollama", "list"],
+) -> bool:
+    """
+    Install a specific Ollama model, ensuring the server is running.
+
+    Args:
+        model_name: Name of the model to install
+        server_manager: OllamaServerManager class or instance
+        installation_cmd: Command to install models
+        ollama_list_cmd: Command to list installed models
+
+    Returns:
+        bool: True if installation is successful
+
+    Raises:
+        Exception: If installation fails or server cannot start
+    """
+    check_ollama_installed()
+
+    with resolve_server_manager(server_manager):
+        logger.info(f"Installing model '{model_name}'...")
+
+        success = auto_install_model(
+            model_name=model_name,
+        )
+
+        if success:
+            logger.info(f"Model '{model_name}' installed successfully")
+            return success
+        logger.error(f"Failed to install model '{model_name}'")
+        raise Exception(f"Failed to install model '{model_name}'")
+
+
+@setup_app.command("model")
+def setup_model(
+    model_name: str = typer.Argument(..., help="Name of the model to install"),
+):
+    """Install a specific model into Ollama."""
+    try:
+        typer.echo(f"Installing model '{model_name}'...")
+
+        install_ollama_model(model_name)
+
+        typer.echo(f"Model '{model_name}' installed successfully!")
+    except Exception as e:
+        typer.echo(f"Error installing model '{model_name}': {e}")
+        logger.error(f"Model installation failed: {e}", exc_info=True)
+        raise typer.Exit(code=1)
+
+
+def get_installed_models(
+    server_manager=OllamaServerManager, command_runner=subprocess.run
+):
+    """
+    Get a list of installed Ollama models.
+
+    Args:
+        server_manager: OllamaServerManager class or instance
+        command_runner: Function to run commands
+
+    Returns:
+        str: Output showing installed models
+
+    Raises:
+        subprocess.SubprocessError: If the command fails
+    """
+    with resolve_server_manager(server_manager):
+        result = command_runner(
+            ["ollama", "list"], capture_output=True, text=True, check=True
+        )
+    return result.stdout
+
+
+@setup_app.command("list-models")
+def list_models():
+    """List all installed Ollama models."""
+    try:
+        output = get_installed_models()
+        typer.echo("Installed models:")
+        typer.echo(output)
+        logger.debug(f"Installed models: \n{output}")
+    except subprocess.SubprocessError:
+        typer.echo("Error: Failed to list models. Is Ollama installed?")
+        raise typer.Exit(code=1)
+    except Exception as e:
+        typer.echo(f"Error: {e}")
+        raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    app()

result_companion/entrypoints/run_rc.py

@@ -0,0 +1,171 @@
+import asyncio
+import time
+from pathlib import Path
+from typing import Optional
+
+from langchain_anthropic import ChatAnthropic
+from langchain_aws import BedrockLLM
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_ollama.llms import OllamaLLM
+from langchain_openai import AzureChatOpenAI, ChatOpenAI
+from pydantic import ValidationError
+
+from result_companion.core.analizers.factory_common import execute_llm_and_get_results
+from result_companion.core.analizers.local.ollama_runner import ollama_on_init_strategy
+from result_companion.core.analizers.models import MODELS
+from result_companion.core.html.html_creator import create_llm_html_log
+from result_companion.core.parsers.config import LLMFactoryModel, load_config
+from result_companion.core.parsers.result_parser import (
+    get_robot_results_from_file_as_dict,
+)
+from result_companion.core.utils.log_levels import LogLevels
+from result_companion.core.utils.logging_config import logger, set_global_log_level
+
+
+def init_llm_with_strategy_factory(
+    config: LLMFactoryModel,
+) -> MODELS:
+    model_type = config.model_type
+    parameters = config.parameters
+
+    model_classes = {
+        "OllamaLLM": (OllamaLLM, ollama_on_init_strategy),
+        "AzureChatOpenAI": (AzureChatOpenAI, None),
+        "BedrockLLM": (BedrockLLM, None),
+        "ChatGoogleGenerativeAI": (ChatGoogleGenerativeAI, None),
+        "ChatOpenAI": (ChatOpenAI, None),
+        "ChatAnthropic": (ChatAnthropic, None),
+    }
+
+    if model_type not in model_classes:
+        raise ValueError(
+            f"Unsupported model type: {model_type} not in {model_classes.keys()}"
+        )
+
+    model_class, strategy = model_classes[model_type]
+    try:
+        return model_class(**parameters), strategy
+    except (TypeError, ValidationError) as e:
+        raise ValueError(
+            f"Invalid parameters for {model_type}: {parameters}, while available parameters are: {model_class.__init__.__annotations__}"
+        ) from e
+
+
+async def _main(
+    output: Path,
+    log_level: LogLevels,
+    config: Optional[Path],
+    report: Optional[str],
+    include_passing: bool,
+    test_case_concurrency: Optional[int] = None,
+    chunk_concurrency: Optional[int] = None,
+    include_tags: Optional[list[str]] = None,
+    exclude_tags: Optional[list[str]] = None,
+) -> bool:
+    set_global_log_level(str(log_level))
+
+    logger.info("Starting Result Companion!")
+    start = time.time()
+    # TODO: move to testable method
+    parsed_config = load_config(config)
+
+    if test_case_concurrency is not None:
+        parsed_config.concurrency.test_case = test_case_concurrency
+    if chunk_concurrency is not None:
+        parsed_config.concurrency.chunk = chunk_concurrency
+
+    # Merge CLI tags with config (CLI takes precedence)
+    final_include = include_tags or parsed_config.test_filter.include_tags or None
+    final_exclude = exclude_tags or parsed_config.test_filter.exclude_tags or None
+
+    # Use RF's native filtering (same as rebot --include/--exclude)
+    # TODO: set output log level from config or cli
+    test_cases = get_robot_results_from_file_as_dict(
+        file_path=output,
+        log_level=LogLevels.DEBUG,
+        include_tags=final_include,
+        exclude_tags=final_exclude,
+    )
+
+    # Filter passing tests (RF doesn't have this natively)
+    should_include_passing = (
+        include_passing or parsed_config.test_filter.include_passing
+    )
+    if not should_include_passing:
+        test_cases = [t for t in test_cases if t.get("status") != "PASS"]
+
+    logger.info(f"Filtered to {len(test_cases)} test cases")
+
+    question_from_config_file = parsed_config.llm_config.question_prompt
+    template = parsed_config.llm_config.prompt_template
+    model, model_init_strategy = init_llm_with_strategy_factory(
+        parsed_config.llm_factory
+    )
+
+    if model_init_strategy:
+        logger.debug(
+            f"Using init strategy: {model_init_strategy} with parameters: {parsed_config.llm_factory.strategy.parameters}"
+        )
+        model_init_strategy(**parsed_config.llm_factory.strategy.parameters)
+
+    logger.debug(f"Prompt template: {template}")
+    logger.debug(f"Question loaded {question_from_config_file=}")
+    prompt_template = ChatPromptTemplate.from_template(template)
+
+    llm_results = await execute_llm_and_get_results(
+        test_cases,
+        parsed_config,
+        prompt_template,
+        model,
+    )
+
+    report_path = report if report else "rc_log.html"
+    if llm_results:
+        model_info = {
+            "model": parsed_config.llm_factory.parameters.get(
+                "model", parsed_config.llm_factory.model_type
+            )
+        }
+        create_llm_html_log(
+            input_result_path=output,
+            llm_output_path=report_path,
+            llm_results=llm_results,
+            model_info=model_info,
+        )
+        logger.info(f"Report created: {Path(report_path).resolve()}")
+
+    stop = time.time()
+    logger.debug(f"Execution time: {stop - start}")
+    return True
+
+
+def run_rc(
+    output: Path,
+    log_level: LogLevels,
+    config: Optional[Path],
+    report: Optional[str],
+    include_passing: bool,
+    test_case_concurrency: Optional[int] = None,
+    chunk_concurrency: Optional[int] = None,
+    include_tags: Optional[list[str]] = None,
+    exclude_tags: Optional[list[str]] = None,
+) -> bool:
+    try:
+        return asyncio.run(
+            _main(
+                output=output,
+                log_level=log_level,
+                config=config,
+                report=report,
+                include_passing=include_passing,
+                test_case_concurrency=test_case_concurrency,
+                chunk_concurrency=chunk_concurrency,
+                include_tags=include_tags,
+                exclude_tags=exclude_tags,
+            )
+        )
+    except Exception:
+        # logging unhandled exceptions to file from asyncio.run
+        logger.critical("Unhandled exception", exc_info=True)
+        raise