symtest-cli 1.3.0__py3-none-any.whl

symtest/file_comparator/script_comparator.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+@file script_comparator.py
+@brief Built-in script comparator – delegates comparison to an external script
+@author Xiaotong Wang
+@date 2025
+"""
+
+import logging
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from .base_comparator import BaseComparator
+from .result import ComparisonResult, Difference
+
+logger = logging.getLogger("symtest.file_comparator.script")
+
+
+class ScriptComparator(BaseComparator):
+    """Run an external script as a comparison step.
+
+    The script is invoked as a subprocess.  By default exit code 0 signals
+    *pass*.  Optionally, ``pass_pattern`` / ``fail_pattern`` regexes can be
+    used to refine the verdict from ``stdout``.
+
+    Configuration example::
+
+        {"type": "script", "script": "analyze_xxx.py",
+         "actual": "...", "baseline": "...", "cwd": ".", "pass_pattern": "PASS",
+         "fail_pattern": "(MISMATCH|FAILED)"}
+    """
+
+    def __init__(
+        self,
+        script: str = "",
+        cwd: Optional[str] = None,
+        args: Optional[List[str]] = None,
+        interpreter: Optional[str] = None,
+        pass_exit_code: int = 0,
+        pass_pattern: Optional[str] = None,
+        fail_pattern: Optional[str] = None,
+        timeout: int = 3600,
+        encoding: str = "utf-8",
+        **kwargs,
+    ):
+        super().__init__(encoding=encoding, **kwargs)
+        self.script = script
+        self.cwd = cwd
+        self.args = args or []
+        self.interpreter = interpreter or sys.executable
+        self.pass_exit_code = pass_exit_code
+        self.pass_pattern = re.compile(pass_pattern) if pass_pattern else None
+        self.fail_pattern = re.compile(fail_pattern) if fail_pattern else None
+        self.timeout = timeout
+
+    # ------------------------------------------------------------------
+    # Abstract method stubs (not used by this comparator)
+    # ------------------------------------------------------------------
+    def read_content(self, file_path, **kwargs):
+        return None
+
+    def compare_content(self, content1, content2):
+        return True, [], False
+
+    # ------------------------------------------------------------------
+    # Core comparison
+    # ------------------------------------------------------------------
+    def compare_files(  # type: ignore[override]
+        self,
+        file1=None,
+        file2=None,
+        **kwargs,
+    ):
+        """Execute the external script and evaluate its output."""
+        result = ComparisonResult(
+            file1=str(file1) if file1 else "",
+            file2=str(file2) if file2 else "",
+        )
+
+        try:
+            cmd = [self.interpreter, self.script, *self.args]
+            if file1:
+                cmd.append(str(file1))
+            if file2:
+                cmd.append(str(file2))
+
+            cwd = self.cwd
+            if cwd and not Path(cwd).is_absolute():
+                cwd = str(Path(cwd).resolve())
+
+            self.logger.info("Executing script: %s", " ".join(cmd))
+            proc = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=self.timeout,
+                cwd=cwd,
+            )
+
+            stdout = proc.stdout or ""
+            stderr = proc.stderr or ""
+            command_output = stdout + ("\n" + stderr if stderr else "")
+            result.command_output = command_output
+
+            # ── Verdict ──
+            exit_ok = proc.returncode == self.pass_exit_code
+
+            pass_match = None
+            fail_match = None
+            if self.pass_pattern:
+                pass_match = self.pass_pattern.search(stdout)
+            if self.fail_pattern:
+                fail_match = self.fail_pattern.search(stdout)
+
+            if fail_match:
+                result.identical = False
+            elif self.pass_pattern and not pass_match:
+                # pass_pattern was set but did not match → fail
+                result.identical = False
+            elif pass_match:
+                result.identical = exit_ok
+            else:
+                result.identical = exit_ok
+
+            if not result.identical:
+                differences: List[Difference] = []
+                if not exit_ok:
+                    differences.append(
+                        Difference(
+                            position="exit_code",
+                            expected=str(self.pass_exit_code),
+                            actual=str(proc.returncode),
+                            diff_type="exit_code_mismatch",
+                        )
+                    )
+                if fail_match:
+                    differences.append(
+                        Difference(
+                            position="stdout pattern",
+                            expected=f"no match for fail_pattern '{self.fail_pattern.pattern}'",
+                            actual=f"matched: {fail_match.group()}",
+                            diff_type="fail_pattern_match",
+                        )
+                    )
+                elif pass_match is None and self.pass_pattern:
+                    differences.append(
+                        Difference(
+                            position="stdout pattern",
+                            expected=f"match for pass_pattern '{self.pass_pattern.pattern}'",
+                            actual="no match found in stdout",
+                            diff_type="pass_pattern_missing",
+                        )
+                    )
+                if not differences:
+                    differences.append(
+                        Difference(
+                            position="stdout",
+                            expected="script determined failure",
+                            actual=stdout[:500] if stdout else "(empty)",
+                            diff_type="script_failure",
+                        )
+                    )
+                result.differences = differences
+
+            return result
+
+        except subprocess.TimeoutExpired:
+            result.error = f"Script timed out after {self.timeout}s: {self.script}"
+            result.identical = False
+            result.command_output = result.error
+            return result
+        except Exception as exc:
+            self.logger.error("Error executing script %s: %s", self.script, exc)
+            result.error = str(exc)
+            result.identical = False
+            result.command_output = result.error
+            return result

symtest/file_comparator/text_comparator.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+@file text_comparator.py
+@brief Text file comparator implementation with line-by-line comparison
+@author Xiaotong Wang
+@date 2025
+"""
+
+import difflib
+import re
+from .base_comparator import BaseComparator
+from .result import Difference
+
+class TextComparator(BaseComparator):
+    """
+    @brief Comparator for text files with line-by-line comparison
+    @details This class implements text file comparison using Python's difflib
+             for detailed difference detection. It supports line and column-based
+             range selection for comparison.
+    """
+    
+    def read_content(self, file_path, start_line=0, end_line=None, start_column=0, end_column=None):
+        """
+        @brief Read text content with specified range
+        @param file_path Path: Path to the text file to read
+        @param start_line int: Starting line number (0-based)
+        @param end_line int: Ending line number (0-based, None for end of file)
+        @param start_column int: Starting column number (0-based)
+        @param end_column int: Ending column number (0-based, None for end of line)
+        @return list: List of text lines within the specified range
+        @throws ValueError: If line or column ranges are invalid
+        @throws UnicodeDecodeError: If file encoding is incorrect
+        @throws FileNotFoundError: If file doesn't exist
+        @throws IOError: If there are other file reading errors
+        """
+        try:
+            self.logger.debug(f"Reading text file: {file_path}")
+            with open(file_path, 'r', encoding=self.encoding) as f:
+                lines = f.readlines()
+                
+            if start_line < 0:
+                raise ValueError("Start line cannot be negative")
+                
+            if end_line is not None:
+                if end_line < start_line:
+                    raise ValueError("End line cannot be before start line")
+                if end_line >= len(lines):
+                    self.logger.warning(f"End line {end_line} exceeds file length {len(lines)}, capping at {len(lines)-1}")
+                    end_line = len(lines) - 1
+            else:
+                end_line = len(lines) - 1
+                
+            if start_line >= len(lines):
+                raise ValueError(f"Start line {start_line} is beyond file length {len(lines)}")
+                
+            selected_lines = lines[start_line:end_line+1]
+            
+            if start_column < 0:
+                raise ValueError("Start column cannot be negative")
+                
+            if start_column > 0 or end_column is not None:
+                self.logger.debug(f"Applying column range: {start_column} to {end_column}")
+                processed_lines = []
+                for line in selected_lines:
+                    if end_column is not None and end_column < start_column:
+                        raise ValueError("End column cannot be before start column")
+                    # Make sure we don't exceed line length
+                    effective_end = end_column
+                    if effective_end is not None and effective_end >= len(line):
+                        effective_end = len(line) - 1
+                    # Apply column range, handle if start_column is beyond line length
+                    if start_column >= len(line):
+                        processed_lines.append("")
+                    else:
+                        processed_lines.append(line[start_column:None if effective_end is None else effective_end+1])
+                return processed_lines
+            
+            return selected_lines
+            
+        except UnicodeDecodeError as e:
+            raise ValueError(f"File encoding error for {file_path}. Try specifying a different encoding. Error: {str(e)}")
+        except FileNotFoundError:
+            raise ValueError(f"File not found: {file_path}")
+        except IOError as e:
+            raise ValueError(f"Error reading file {file_path}: {str(e)}")
+    
+    def compare_content(self, content1, content2):
+        """
+        @brief Compare text content and return detailed differences
+        @param content1 list: First list of text lines to compare
+        @param content2 list: Second list of text lines to compare
+        @return tuple: (bool, list, bool) - (identical, differences, truncated)
+        @details Uses difflib to generate a detailed comparison of the text content.
+                 Returns a tuple containing a boolean indicating if the content is identical
+                 and a list of Difference objects describing any differences found.
+                 Limits the number of differences reported to 10 to avoid overwhelming output.
+        """
+        self.logger.debug(f"Comparing text content")
+        
+        if content1 == content2:
+            return True, [], False
+            
+        differences = []
+        
+        # Use difflib for more detailed comparison
+        diff = list(difflib.unified_diff(content1, content2, n=0))
+        
+        # Process the diff output to create structured differences
+        line_diffs = []
+        for line in diff[2:]:  # Skip the first two header lines
+            if line.startswith('@@'):
+                # Parse hunk header for real line numbers: @@ -start1,c1 +start2,c2 @@
+                match = re.match(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@', line)
+                if match:
+                    line_diffs.append(('hunk', (int(match.group(1)), int(match.group(3)))))
+                continue
+            elif line.startswith('-'):
+                line_diffs.append(('remove', line[1:]))
+            elif line.startswith('+'):
+                line_diffs.append(('add', line[1:]))
+            else:
+                line_diffs.append(('context', line[1:]))
+        
+        # Convert diff to our difference format
+        line_num1 = 0
+        line_num2 = 0
+        for i, (action, line) in enumerate(line_diffs):
+            if action == 'hunk':
+                # Reset to real line numbers from the hunk header (convert 1-based to 0-based)
+                line_num1 = line[0] - 1
+                line_num2 = line[1] - 1
+                continue
+            elif action == 'remove':
+                # Look ahead for a corresponding 'add'
+                add_match = None
+                for j in range(i+1, min(i+5, len(line_diffs))):
+                    if line_diffs[j][0] == 'add':
+                        add_match = line_diffs[j][1]
+                        del line_diffs[j]
+                        break
+                
+                if add_match is not None:
+                    # Content difference
+                    differences.append(Difference(
+                        position=f"line {line_num1+1}",
+                        expected=line,
+                        actual=add_match,
+                        diff_type="content"
+                    ))
+                else:
+                    # Missing line
+                    differences.append(Difference(
+                        position=f"line {line_num1+1}",
+                        expected=line,
+                        actual=None,
+                        diff_type="missing"
+                    ))
+                line_num1 += 1
+                
+            elif action == 'add':
+                # Extra line
+                differences.append(Difference(
+                    position=f"line {line_num2+1}",
+                    expected=None,
+                    actual=line,
+                    diff_type="extra"
+                ))
+                line_num2 += 1
+                
+            elif action == 'context':
+                line_num1 += 1
+                line_num2 += 1
+        
+        # Limit the number of differences reported
+        max_diffs = 10
+        truncated = len(differences) > max_diffs
+        if truncated:
+            differences = differences[:max_diffs]
+            
+        return False, differences, truncated

symtest/file_comparator/xml_comparator.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+@file xml_comparator.py
+@brief XML file comparator implementation with structural comparison
+@author Xiaotong Wang
+@date 2025
+"""
+
+import xml.etree.ElementTree as ET
+from .text_comparator import TextComparator
+from .result import Difference
+
+class XmlComparator(TextComparator):
+    """
+    @brief Comparator for XML files with structural comparison
+    @details This class extends TextComparator to provide specialized XML comparison
+             capabilities, including:
+             - Tag comparison
+             - Attribute comparison
+             - Text content comparison
+             - Child element comparison
+    """
+    
+    def read_content(self, file_path, start_line=0, end_line=None, start_column=0, end_column=None):
+        """
+        @brief Read and parse XML content from file
+        @param file_path Path: Path to the XML file
+        @param start_line int: Starting line number
+        @param end_line int: Ending line number
+        @param start_column int: Starting column number
+        @param end_column int: Ending column number
+        @return ET.Element: Parsed XML element tree
+        @throws ValueError: If XML is invalid
+        """
+        # First read the file as text
+        text_content = super().read_content(file_path, start_line, end_line, start_column, end_column)
+        
+        # Join the lines
+        xml_text = ''.join(text_content)
+        try:
+            return ET.fromstring(xml_text)
+        except ET.ParseError as e:
+            raise ValueError(f"Invalid XML in {file_path}: {str(e)}")
+    
+    def compare_content(self, content1, content2):
+        """
+        @brief Compare XML content structurally
+        @param content1 ET.Element: First XML element to compare
+        @param content2 ET.Element: Second XML element to compare
+        @return tuple: (bool, list, bool) - (identical, differences, truncated)
+        @details Performs structural comparison of XML elements, including tags,
+                 attributes, text content, and child elements
+        """
+        # Convert back to strings for comparison
+        xml_str1 = ET.tostring(content1, encoding='unicode')
+        xml_str2 = ET.tostring(content2, encoding='unicode')
+        
+        if xml_str1 == xml_str2:
+            return True, [], False
+            
+        # Use a recursive function to find differences in XML structures
+        differences = []
+        self._compare_elements(content1, content2, "", differences)
+        
+        return False, differences, False
+    
+    def _compare_elements(self, elem1, elem2, path, differences, max_diffs=10):
+        """
+        @brief Recursively compare XML elements and collect differences
+        @param elem1 ET.Element: First XML element to compare
+        @param elem2 ET.Element: Second XML element to compare
+        @param path str: Current path in the XML structure
+        @param differences list: List to store found differences
+        @param max_diffs int: Maximum number of differences to report
+        @details Compares XML elements recursively, checking for:
+                 - Tag mismatches
+                 - Missing or extra attributes
+                 - Text content differences
+                 - Child element count mismatches
+                 - Child element differences
+        """
+        if len(differences) >= max_diffs:
+            return
+            
+        # Compare tags
+        if elem1.tag != elem2.tag:
+            differences.append(Difference(
+                position=path or "/",
+                expected=elem1.tag,
+                actual=elem2.tag,
+                diff_type="tag_mismatch"
+            ))
+            return  # If tags don't match, don't compare further
+            
+        # Compare attributes
+        attrib1 = set(elem1.attrib.items())
+        attrib2 = set(elem2.attrib.items())
+        
+        for attr, value in attrib1 - attrib2:
+            differences.append(Difference(
+                position=f"{path}/@{attr}" if path else f"/@{attr}",
+                expected=value,
+                actual="missing attribute",
+                diff_type="missing_attribute"
+            ))
+            if len(differences) >= max_diffs:
+                return
+                
+        for attr, value in attrib2 - attrib1:
+            differences.append(Difference(
+                position=f"{path}/@{attr}" if path else f"/@{attr}",
+                expected="missing attribute",
+                actual=value,
+                diff_type="extra_attribute"
+            ))
+            if len(differences) >= max_diffs:
+                return
+        
+        # Compare text content if leaf nodes
+        if len(elem1) == 0 and len(elem2) == 0:
+            text1 = elem1.text.strip() if elem1.text else ""
+            text2 = elem2.text.strip() if elem2.text else ""
+            
+            if text1 != text2:
+                differences.append(Difference(
+                    position=path or "/",
+                    expected=text1,
+                    actual=text2,
+                    diff_type="text_mismatch"
+                ))
+                return
+                
+        # Compare children elements
+        children1 = list(elem1)
+        children2 = list(elem2)
+        
+        if len(children1) != len(children2):
+            differences.append(Difference(
+                position=path or "/",
+                expected=f"{len(children1)} child elements",
+                actual=f"{len(children2)} child elements",
+                diff_type="children_count_mismatch"
+            ))
+            
+        # Compare matching children
+        for i, (child1, child2) in enumerate(zip(children1, children2)):
+            new_path = f"{path}/{child1.tag}[{i}]" if path else f"/{child1.tag}[{i}]"
+            self._compare_elements(child1, child2, new_path, differences, max_diffs)

symtest/logging_config.py

@@ -0,0 +1,66 @@
+"""
+Central logging configuration for the CLI test framework.
+
+Provides a unified ``get_logger(name)`` helper that returns a logger
+under the ``symtest`` namespace.  By default only a
+``NullHandler`` is attached — no output is produced when the package
+is imported as a library.  Applications that want console logging
+should call ``setup_console_logging()`` (the CLI entry point does
+this automatically).
+
+Replaces the previous ad-hoc ``print()`` / ``_print_lock`` pattern.
+The ``logging`` module is already thread-safe, so the manual locking
+is no longer needed.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+DEFAULT_FORMAT = "%(levelname)-7s %(name)-35s %(message)s"
+
+# ── package-level logger (used as parent for all child loggers) ──────────────
+_logger = logging.getLogger("symtest")
+_logger.setLevel(logging.DEBUG)          # let handlers decide; do not block
+
+# ── default: NullHandler (polite library behaviour) ──────────────────────────
+_logger.addHandler(logging.NullHandler())
+
+
+def setup_console_logging(level: int = logging.INFO) -> None:
+    """Attach a console handler (stderr) for CLI usage.
+
+    Replaces any existing NullHandler with a StreamHandler writing
+    to ``sys.stderr`` so that logs never pollute the ``stdout``
+    stream (which is reserved for machine-readable output such as
+    ``--output-format json``).
+    """
+    # Remove any NullHandler so we don't have duplicate handlers.
+    _logger.handlers = [
+        h for h in _logger.handlers
+        if not isinstance(h, logging.NullHandler)
+    ]
+
+    # Only add if no real handler is already attached (idempotent).
+    if not _logger.handlers:
+        _handler = logging.StreamHandler(sys.stderr)
+        _handler.setLevel(level)
+        _handler.setFormatter(logging.Formatter(DEFAULT_FORMAT))
+        _logger.addHandler(_handler)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Return a child logger under the ``symtest`` namespace.
+
+    Usage::
+
+        from symtest.logging_config import get_logger
+        logger = get_logger(__name__)
+        logger.info("Hello")
+    """
+    # Ensure the logger is a child of symtest so it inherits
+    # the handler (unless the user removes / replaces it).
+    if not name.startswith("symtest"):
+        name = "symtest." + name
+    return logging.getLogger(name)

symtest/runners/__init__.py

@@ -0,0 +1,15 @@
+"""
+Test runners for the CLI Testing Framework
+"""
+
+from .json_runner import JSONRunner
+from .parallel_json_runner import ParallelJSONRunner
+from .parallel_yaml_runner import ParallelYAMLRunner
+from .yaml_runner import YAMLRunner
+
+__all__ = [
+    'JSONRunner',
+    'ParallelJSONRunner',
+    'ParallelYAMLRunner',
+    'YAMLRunner'
+]

symtest/runners/config_runner.py

@@ -0,0 +1,96 @@
+"""Generic config-driven test runner with injectable config loader.
+
+The only difference between JSON and YAML runners was the ``load`` call.
+This module extracts the common logic into ``ConfigRunner``, accepting a
+``config_loader`` callable.  ``JSONRunner`` and ``YAMLRunner`` are now thin
+wrappers that inject ``json.load`` / ``yaml.safe_load`` respectively.
+"""
+import sys
+import logging
+from typing import Optional, Dict, Any, Callable, BinaryIO
+
+from ..core.base_runner import BaseRunner
+from ..core.config_loader import parse_test_cases
+from ..config.inheritance_expander import resolve_inheritance, apply_variables
+from ..core.test_case import TestCase
+from ..core.execution import execute_single_test_case
+from ..core.types import TestCaseData
+from ..utils.path_resolver import PathResolver
+from ..config.import_expander import expand_imports
+
+logger = logging.getLogger("symtest.runners.config_runner")
+
+
+class ConfigRunner(BaseRunner):
+    """Generic sequential test runner.
+
+    Instead of hardcoding ``json.load`` / ``yaml.safe_load``, subclasses (or
+    direct callers) inject a ``config_loader`` – any callable that accepts an
+    open file-like object and returns a ``dict``.
+    """
+
+    def __init__(self, config_file: str = "test_cases.json",
+                 workspace: Optional[str] = None,
+                 config_loader: Optional[Callable[[BinaryIO], Dict[str, Any]]] = None,
+                 variables: Optional[Dict[str, Any]] = None,
+                 **kwargs):
+        super().__init__(config_file, workspace, **kwargs)
+        self._config_loader = config_loader
+        self._variables = variables or {}
+        # Backward-compatible attribute for tests that patch path_resolver
+        self.path_resolver = PathResolver(self.workspace)
+
+    def load_test_cases(self) -> None:
+        """Load test cases from config file using the injected loader."""
+        if self._config_loader is None:
+            raise RuntimeError(
+                "config_loader must be set before loading test cases"
+            )
+        try:
+            with open(self.config_path, 'r', encoding='utf-8') as f:
+                config = self._config_loader(f)
+
+            # Expand import references (no-op if none present)
+            config = expand_imports(config, self.config_path)
+            # Resolve extends inheritance (no-op if none present)
+            config = resolve_inheritance(config)
+            # Per-case + global variable substitution
+            config = apply_variables(config, self._variables)
+
+            self.load_setup_from_config(config)
+            self.test_cases = parse_test_cases(
+                config, self.workspace, self.path_resolver,
+            )
+
+            logger.info("Successfully loaded %d test cases",
+                        len(self.test_cases))
+        except Exception as e:
+            sys.exit(f"Failed to load configuration file: {str(e)}")
+
+    def run_single_test(self, case: TestCase) -> Dict[str, Any]:
+        if case.steps:
+            return self._run_sequence(case)
+
+        case_data = case.to_execution_dict()
+
+        command_preview = (
+            f"{case_data['command']} {' '.join(case_data['args'])}".strip()
+        )
+        logger.info("  Executing command: %s", command_preview)
+
+        result = execute_single_test_case(
+            case_data,
+            str(self.workspace) if self.workspace else None,
+            update_baseline=self.update_baseline,
+            error_analysis=self.error_analysis,
+        )
+
+        if result["output"].strip():
+            logger.debug("  Command output:")
+            for line in result["output"].splitlines():
+                logger.debug("    %s", line)
+
+        if result["status"] != "passed" and result.get("message"):
+            logger.error("  Error: %s", result["message"])
+
+        return result