result-companion 0.0.1__py3-none-any.whl

result_companion/core/html/html_creator.py

@@ -0,0 +1,179 @@
+from pathlib import Path
+from typing import Dict
+
+from robot.api import ExecutionResult
+from robot.reporting.resultwriter import ResultWriter
+
+from result_companion.core.html.llm_injector import LLMDataInjector
+from result_companion.core.results.visitors import UniqueNameResultVisitor
+
+
+def create_llm_html_log(
+    input_result_path: Path | str,
+    llm_output_path: Path | str,
+    llm_results: Dict[str, str],
+    model_info: Dict[str, str] = None,
+) -> None:
+    """Create HTML log with LLM data embedded in JS model.
+
+    Args:
+        input_result_path: Path to Robot Framework output.xml.
+        llm_output_path: Path for generated HTML report.
+        llm_results: Mapping of test names to LLM analysis.
+        model_info: Optional model information.
+    """
+    results = ExecutionResult(str(input_result_path))
+
+    results.visit(UniqueNameResultVisitor())
+    results.visit(LLMDataInjector(llm_results, model_info))
+
+    writer = ResultWriter(results)
+    writer.write_results(report=None, log=str(llm_output_path))
+
+    _inject_llm_ui(Path(llm_output_path))
+
+
+def _inject_llm_ui(html_path: Path) -> None:
+    """Add JavaScript to display LLM results per test."""
+    script = """
+<style>
+.llm-section { margin: 12px 0; border: 1px solid var(--secondary-color); border-radius: 6px; overflow: hidden; }
+.llm-header { padding: 10px 16px; background: var(--primary-color); color: var(--text-color); cursor: pointer; display: flex; justify-content: space-between; align-items: center; user-select: none; }
+.llm-header:hover { background: var(--secondary-color); }
+.llm-chevron { transition: transform 0.2s; font-size: 12px; }
+.llm-chevron.collapsed { transform: rotate(-90deg); }
+.llm-content { padding: 16px; background: var(--background-color); border-top: 1px solid var(--secondary-color); max-height: 500px; overflow-y: auto; display: none; position: relative; }
+.llm-content h2 { color: var(--link-color); font-size: 14px; margin-top: 12px; margin-bottom: 8px; }
+.llm-model { font-size: 11px; opacity: 0.7; margin-left: 8px; }
+.llm-copy { position: absolute; top: 8px; right: 8px; background: var(--secondary-color); color: var(--text-color); border: 1px solid var(--primary-color); padding: 4px 8px; border-radius: 4px; cursor: pointer; font-size: 12px; }
+.llm-copy:hover { background: var(--primary-color); }
+.llm-copy.copied { background: var(--pass-color); color: white; }
+.test.fail .llm-header { border-left: 4px solid var(--fail-color); }
+.test.pass .llm-header { border-left: 4px solid var(--pass-color); }
+</style>
+<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
+<script>
+$(function() {
+    var llmData = null;
+    var modelInfo = null;
+    var processed = new Set();
+
+    // Hide metadata row immediately
+    $('th').filter(function() { return $(this).text().indexOf('__llm_results') !== -1; }).parent().hide();
+
+    // Get LLM data
+    try {
+        var meta = window.testdata.suite().metadata;
+        for (var i in meta) {
+            if (meta[i][0] === '__llm_results') {
+                var div = document.createElement('div');
+                div.innerHTML = meta[i][1];
+                var decoded = div.textContent || div.innerText || '';
+                var data = JSON.parse(decoded);
+
+                if (data.results) {
+                    llmData = data.results;
+                    modelInfo = data.model;
+                } else {
+                    llmData = data;
+                }
+                break;
+            }
+        }
+    } catch(e) {
+        console.error('Failed to load LLM data:', e);
+    }
+
+    // Process test element
+    function process(test) {
+        var id = test.attr('id');
+        if (!id || processed.has(id) || !llmData) return;
+
+        var name = test.find('.element-header .name').first().text().trim();
+
+        if (llmData[name]) {
+            processed.add(id);
+            var modelBadge = modelInfo ? '<span class="llm-model">' + modelInfo.model + '</span>' : '';
+            var html = '<div class="llm-section">' +
+                '<div class="llm-header">' +
+                    '<div>🤖 AI Analysis ' + modelBadge + '</div>' +
+                    '<span class="llm-chevron collapsed">▼</span>' +
+                '</div>' +
+                '<div class="llm-content">' +
+                    '<button class="llm-copy">Copy</button>' +
+                    marked.parse(llmData[name]) +
+                '</div></div>';
+            test.find('.children').first().append(html);
+
+            // Toggle with animation
+            test.find('.llm-header').click(function() {
+                var content = $(this).next();
+                var chevron = $(this).find('.llm-chevron');
+                content.slideToggle(200);
+                chevron.toggleClass('collapsed');
+            });
+
+            // Copy button
+            test.find('.llm-copy').click(function(e) {
+                e.stopPropagation();
+                var btn = $(this);
+                navigator.clipboard.writeText(llmData[name]);
+                btn.text('✓ Copied').addClass('copied');
+                setTimeout(function() {
+                    btn.text('Copy').removeClass('copied');
+                }, 2000);
+            });
+        }
+    }
+
+    // Process all tests periodically
+    setInterval(function() {
+        $('.test').each(function() { process($(this)); });
+    }, 1000);
+});
+</script>
+"""
+    html = html_path.read_text()
+    html_path.write_text(html.replace("</body>", f"{script}\n</body>"))
+
+
+if __name__ == "__main__":
+    # TODO: remove this test code
+    REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+    input_result_path = REPO_ROOT / ".." / "examples" / "run_test" / "output.xml"
+    multiline_another = """**General Idea Behind Test Case**
+This test case is designed to execute a SQL query on a database and validate the results.
+
+**Flow**
+
+* The test connects to the database using a provided connection string.
+* It logs a message indicating that the query is being executed.
+* The test executes the SQL query.
+* If the query fails, it logs an error message and raises an exception.
+* Finally, the test checks if the result of the query matches an expected result.
+
+**Failure Root Cause**
+The root cause of the failure is that the database connection string is invalid, causing a "Connection Timeout" error. This prevents the test from successfully executing the SQL query and comparing its results to the expected result.
+
+**Potential Fixes**
+
+* Verify that the provided connection string is correct and properly formatted.
+* Use a valid and existing database connection string for testing purposes.
+* Consider using environment variables or configuration files to store sensitive information like database credentials, making it easier to manage and rotate them.
+
+```python
+import os
+os.environ["DB_CONNECTION_STRING"] = "valid_connection_string"
+```
+"""
+
+    multiline_html_response = """<div> something here </div>
+                                    <div> deeper something here </div>"""  # .replace("\n", " \\ \n")
+    create_llm_html_log(
+        input_result_path=input_result_path,
+        llm_results={
+            "Test Neasted Test Case": multiline_html_response,
+            "Ollama Local Model Run Should Succede": multiline_another,
+        },
+        llm_output_path="test_llm_full_log.html",
+    )

result_companion/core/html/llm_injector.py

@@ -0,0 +1,20 @@
+import json
+from typing import Dict
+
+from robot.result.visitor import ResultVisitor
+
+
+class LLMDataInjector(ResultVisitor):
+    """Injects LLM results directly into test data."""
+
+    def __init__(self, llm_results: Dict[str, str], model_info: Dict[str, str] = None):
+        self.llm_results = llm_results
+        self.model_info = model_info
+
+    def end_result(self, result):
+        """Store LLM data as global metadata."""
+        if result.suite and self.llm_results:
+            data = {"results": self.llm_results}
+            if self.model_info:
+                data["model"] = self.model_info
+            result.suite.metadata["__llm_results"] = json.dumps(data)

result_companion/core/parsers/config.py

@@ -0,0 +1,256 @@
+import os
+import re
+from enum import Enum
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel, Field, SecretStr, ValidationError, model_serializer
+
+from result_companion.core.utils.logging_config import logger
+
+
+class ModelType(str, Enum):
+    LOCAL = "local"
+    REMOTE = "remote"
+
+
+class TokenizerTypes(str, Enum):
+    AZURE_OPENAI = "azure_openai_tokenizer"
+    OLLAMA = "ollama_tokenizer"
+    BEDROCK = "bedrock_tokenizer"
+    GOOGLE = "google_tokenizer"
+    OPENAI = "openai_tokenizer"
+    ANTHROPIC = "anthropic_tokenizer"
+
+
+class CustomEndpointModel(BaseModel):
+    azure_deployment: str = Field(min_length=5, description="Azure deployment URL.")
+    azure_endpoint: str
+    openai_api_version: str = Field(
+        min_length=5, description="OpenAI API version.", default="2023-03-15-preview"
+    )
+    openai_api_type: str = Field(
+        min_length=5, description="OpenAI API type.", default="azure"
+    )
+    openai_api_key: SecretStr = Field(min_length=5, description="OpenAI API key.")
+
+
+class ChunkingPromptsModel(BaseModel):
+    chunk_analysis_prompt: str = Field(
+        min_length=5, description="Prompt for analyzing individual chunks."
+    )
+    final_synthesis_prompt: str = Field(
+        min_length=5, description="Prompt for synthesizing chunk summaries."
+    )
+
+
+class LLMConfigModel(BaseModel):
+    question_prompt: str = Field(min_length=5, description="User prompt.")
+    prompt_template: str = Field(
+        min_length=5, description="Template for LLM ChatPromptTemplate."
+    )
+    chunking: ChunkingPromptsModel
+    model_type: ModelType = Field(
+        default=ModelType.LOCAL,
+        description=f"Which type of llm model runners to use {[el.name for el in ModelType]}",
+    )
+
+
+class LLMInitStrategyModel(BaseModel):
+    parameters: dict = Field(default={}, description="Strategy parameters.")
+
+
+class LLMFactoryModel(BaseModel):
+    model_type: str = Field(min_length=5, description="Model type.")
+    parameters: dict = Field(default={}, description="Model parameters.")
+    strategy: LLMInitStrategyModel = Field(
+        description="Strategy to run on init.", default_factory=LLMInitStrategyModel
+    )
+
+    def _get_masked_params(self) -> dict:
+        """Returns parameters dict with sensitive values masked."""
+        sensitive_keys = {"api_key", "token", "password", "secret", "auth"}
+        masked = {}
+        for key, value in self.parameters.items():
+            if any(s in key.lower() for s in sensitive_keys):
+                masked[key] = "***REDACTED***"
+            else:
+                masked[key] = value
+        return masked
+
+    def __repr__(self) -> str:
+        """Returns string representation with masked sensitive fields."""
+        return (
+            f"LLMFactoryModel(model_type={self.model_type!r}, "
+            f"parameters={self._get_masked_params()!r}, "
+            f"strategy={self.strategy!r})"
+        )
+
+    @model_serializer
+    def _mask_sensitive_fields(self) -> dict:
+        """Masks sensitive fields in parameters dict for serialization."""
+        return {
+            "model_type": self.model_type,
+            "parameters": self._get_masked_params(),
+            "strategy": self.strategy,
+        }
+
+
+class TokenizerModel(BaseModel):
+    tokenizer: TokenizerTypes
+    max_content_tokens: int = Field(ge=0, description="Max content tokens.")
+
+
+class ConcurrencyModel(BaseModel):
+    test_case: int = Field(
+        default=1, ge=1, description="Test cases processed in parallel."
+    )
+    chunk: int = Field(
+        default=1, ge=1, description="Chunks processed in parallel per test case."
+    )
+
+
+class TestFilterModel(BaseModel):
+    """Test filtering config - passed to RF's native result.configure()."""
+
+    include_tags: list[str] = Field(default=[], description="RF --include patterns.")
+    exclude_tags: list[str] = Field(default=[], description="RF --exclude patterns.")
+    include_passing: bool = Field(default=False, description="Include PASS tests.")
+
+
+class DefaultConfigModel(BaseModel):
+    version: float
+    llm_config: LLMConfigModel
+    llm_factory: LLMFactoryModel
+    tokenizer: TokenizerModel
+    concurrency: ConcurrencyModel = Field(default_factory=ConcurrencyModel)
+    test_filter: TestFilterModel = Field(default_factory=TestFilterModel)
+
+
+class CustomModelEndpointConfig(DefaultConfigModel):
+    custom_endpoint: CustomEndpointModel
+
+
+class ConfigLoader:
+    def __init__(
+        self,
+        default_config_file: Path | None = None,
+    ):
+        self.default_config_file = default_config_file
+
+    @staticmethod
+    def _read_yaml_file(file_path: Path) -> dict:
+        with open(file_path, "r") as file:
+            return yaml.safe_load(file)
+
+    @staticmethod
+    def _expand_env_vars(value):
+        """Expand environment variables in a string using ${VAR} syntax."""
+        if isinstance(value, str) and "${" in value and "}" in value:
+            pattern = re.compile(r"\${([^}^{]+)}")
+            matches = pattern.findall(value)
+            for match in matches:
+                env_var = os.environ.get(match)
+                if env_var:
+                    value = value.replace(f"${{{match}}}", env_var)
+                else:
+                    logger.warning(f"Environment variable '{match}' not found")
+            return value
+        return value
+
+    def _process_env_vars(self, data):
+        """Recursively process environment variables in the configuration data."""
+        if isinstance(data, dict):
+            return {k: self._process_env_vars(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self._process_env_vars(item) for item in data]
+        else:
+            return self._expand_env_vars(data)
+
+    def load_config(self, user_config_file: Path = None) -> DefaultConfigModel:
+        """Load and validate the YAML configuration file, with defaults."""
+        default_config = self._read_yaml_file(self.default_config_file)
+        # Process environment variables in default config
+        default_config = self._process_env_vars(default_config)
+
+        if user_config_file:
+            user_config = self._read_yaml_file(user_config_file)
+            # Process environment variables in user config
+            user_config = self._process_env_vars(user_config)
+        else:
+            logger.info(
+                "User configuration not found or not provided. Using default configuration!"
+            )
+            logger.debug({self.default_config_file})
+            user_config = {}
+
+        # TODO: improve unpacking
+        config_data = (
+            {
+                "version": default_config.get("version"),
+                "llm_config": {
+                    **default_config.get("llm_config", {}),
+                    **user_config.get("llm_config", {}),
+                },
+                "llm_factory": {
+                    **default_config.get("llm_factory", {}),
+                    **user_config.get("llm_factory", {}),
+                },
+                "tokenizer": {
+                    **default_config.get("tokenizer", {}),
+                    **user_config.get("tokenizer", {}),
+                },
+                "concurrency": {
+                    **default_config.get("concurrency", {}),
+                    **user_config.get("concurrency", {}),
+                },
+                "test_filter": {
+                    **default_config.get("test_filter", {}),
+                    **user_config.get("test_filter", {}),
+                },
+            }
+            if user_config_file
+            else default_config
+        )
+        try:
+            validated_config = DefaultConfigModel(**config_data)
+        except ValidationError as e:
+            logger.error(f"Configuration validation failed:\n{e}")
+            raise
+        return validated_config
+
+
+def load_config(config_path: Path | None = None) -> DefaultConfigModel:
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    config_file_path = os.path.join(current_dir, "..", "configs", "default_config.yaml")
+
+    config_loader = ConfigLoader(default_config_file=config_file_path)
+    config = config_loader.load_config(user_config_file=config_path)
+    logger.debug(f"{config=}")
+    return config
+
+
+# TODO: remove this code
+# Example usage in a CLI application
+if __name__ == "__main__":
+    import argparse
+    from pathlib import Path
+
+    # Define default config path (assumes it's in the package directory)
+    PACKAGE_DIR = Path(__file__).resolve().parent
+    DEFAULT_CONFIG_PATH = PACKAGE_DIR / "config" / "config.yaml"
+
+    parser = argparse.ArgumentParser(description="CLI Application with Config")
+    parser.add_argument(
+        "--config", type=str, help="Path to the YAML configuration file (optional)."
+    )
+    args = parser.parse_args()
+
+    config_loader = ConfigLoader(default_config_file=str(DEFAULT_CONFIG_PATH))
+    try:
+        config = config_loader.load_config(user_config_file=args.config)
+        print(f"Config loaded successfully: {config}")
+        # Use the `config.prompt` in your CLI application logic
+        print(f"Prompt: {config.prompt}")
+    except Exception as e:
+        print(f"Failed to load configuration: {e}")

result_companion/core/parsers/result_parser.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+
+from robot.api import ExecutionResult
+
+from result_companion.core.results.visitors import UniqueNameResultVisitor
+from result_companion.core.utils.log_levels import LogLevels
+from result_companion.core.utils.logging_config import logger
+
+
+def search_for_test_cases(
+    tests: dict | list, accumulated: list | None = None
+) -> list[dict]:
+    """Recursively extracts test cases from nested suite structure."""
+    if accumulated is None:
+        accumulated = []
+    if isinstance(tests, list):
+        for el in tests:
+            search_for_test_cases(el, accumulated)
+    elif isinstance(tests, dict):
+        if tests.get("tests"):
+            accumulated.extend(tests["tests"])
+        elif tests.get("suites"):
+            search_for_test_cases(tests["suites"], accumulated)
+    return accumulated
+
+
+def remove_redundant_fields(data: list[dict]) -> list[dict]:
+    fields_to_remove = [
+        "elapsed_time",
+        "lineno",
+        "owner",
+        "start_time",
+        "html",
+        "type",
+        "assign",
+        "level",
+        "timestamp",
+    ]
+
+    if isinstance(data, dict):
+        # Remove fields from the current dictionary
+        for field in fields_to_remove:
+            data.pop(field, None)
+
+        # Recursively process child dictionaries
+        for key, value in data.items():
+            if isinstance(value, dict):
+                data[key] = remove_redundant_fields(value)
+            elif isinstance(value, list):
+                data[key] = [
+                    remove_redundant_fields(item) if isinstance(item, dict) else item
+                    for item in value
+                ]
+
+    elif isinstance(data, list):
+        # Recursively process child dictionaries in the list
+        return [
+            remove_redundant_fields(item) if isinstance(item, dict) else item
+            for item in data
+        ]
+
+    return data
+
+
+def get_robot_results_from_file_as_dict(
+    file_path: Path,
+    log_level: LogLevels,
+    include_tags: list[str] | None = None,
+    exclude_tags: list[str] | None = None,
+) -> list[dict]:
+    """Parses RF output.xml and returns test cases as dicts.
+
+    Uses RF's native filtering via result.configure() - same as rebot.
+
+    Args:
+        file_path: Path to output.xml.
+        log_level: Log level for parsing.
+        include_tags: RF tag patterns to include (e.g., ['smoke*', 'critical']).
+        exclude_tags: RF tag patterns to exclude (e.g., ['wip', 'bug-*']).
+
+    Returns:
+        List of test case dictionaries.
+    """
+    logger.debug(f"Getting robot results from {file_path}")
+    result = ExecutionResult(file_path)
+
+    # Use RF's native filtering (same as rebot --include/--exclude)
+    suite_config = {}
+    if include_tags:
+        suite_config["include_tags"] = include_tags
+    if exclude_tags:
+        suite_config["exclude_tags"] = exclude_tags
+    if suite_config:
+        result.configure(suite_config=suite_config)
+        logger.debug(f"Applied RF native filtering: {suite_config}")
+
+    result.visit(UniqueNameResultVisitor())
+    all_results = result.suite.to_dict()
+    all_results = search_for_test_cases(all_results)
+    all_results = remove_redundant_fields(all_results)
+    return all_results

result_companion/core/results/visitors.py

@@ -0,0 +1,34 @@
+from robot.api import ResultVisitor
+
+from result_companion.core.utils.logging_config import logger
+
+
+# TODO: workaround to fix potentail problem with exposing llm results to invalid test cases in log.html
+class UniqueNameResultVisitor(ResultVisitor):
+    """Custom visitor that ensures unique test case names by appending IDs to duplicates."""
+
+    def __init__(self):
+        super().__init__()
+        self.test_names = {}
+
+    def start_test(self, test):
+        """Called when a test is encountered during traversal."""
+        if test.name in self.test_names:
+            self.test_names[test.name] += 1
+        else:
+            self.test_names[test.name] = 1
+
+    def end_suite(self, suite):
+        """Called when suite processing is complete."""
+        for test in suite.tests:
+            if test.name in self.test_names and self.test_names[test.name] > 1:
+                logger.debug(f"Renaming test '{test.name}' to '{test.name} {test.id}'")
+                test.name = f"{test.name} {test.id}"
+
+        for child_suite in suite.suites:
+            for test in child_suite.tests:
+                if test.name in self.test_names and self.test_names[test.name] > 1:
+                    logger.debug(
+                        f"Renaming test '{test.name}' to '{test.name} {test.id}'"
+                    )
+                    test.name = f"{test.name} {test.id}"

result_companion/core/utils/log_levels.py

@@ -0,0 +1,23 @@
+from enum import Enum
+from typing import Any
+
+
+class LogLevels(str, Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+    def __str__(self) -> str:
+        return self.name
+
+    @classmethod
+    def from_str(cls, value) -> Any:
+        try:
+            return cls[value.upper()]
+        except KeyError as err:
+            _msg = (
+                f"Values available: {[e.name for e in cls]}, while provided: {value!r}"
+            )
+            raise ValueError(_msg) from err