symtest-cli 1.3.0__py3-none-any.whl

symtest/core/__init__.py

@@ -0,0 +1,15 @@
+"""
+Core components for the CLI Testing Framework
+"""
+
+from .base_runner import BaseRunner
+from .parallel_runner import ParallelRunner
+from .test_case import TestCase
+from .assertions import Assertions
+
+__all__ = [
+    'BaseRunner',
+    'ParallelRunner', 
+    'TestCase',
+    'Assertions'
+]

symtest/core/assertions.py

@@ -0,0 +1,253 @@
+import os
+import re
+import shutil
+import logging
+from typing import Any, Dict, List, Optional, Pattern
+
+from ..file_comparator.factory import ComparatorFactory
+
+logger = logging.getLogger("symtest.core.assertions")
+
+
+def _detect_file_type(file_path: str) -> str:
+    """Auto-detect comparator type from file extension."""
+    ext = os.path.splitext(file_path)[1].lower()
+    _type_map = {
+        '.h5': 'h5', '.hdf5': 'h5', '.hdf': 'h5',
+        '.json': 'json',
+        '.csv': 'csv', '.tsv': 'csv',
+        '.xml': 'xml', '.html': 'xml', '.htm': 'xml',
+        '.txt': 'text', '.log': 'text', '.out': 'text', '.py': 'text',
+    }
+    if ext in _type_map:
+        return _type_map[ext]
+    # For unknown extensions, use binary comparator
+    return 'binary'
+
+
+class ValidationError(AssertionError):
+    """Structured assertion failure carrying failure_kind and detailed data.
+
+    Extends ``AssertionError`` for backward compatibility with existing
+    ``except AssertionError`` catch blocks, while adding structured fields
+    that reporters and AI consumers can use directly.
+    """
+
+    def __init__(
+        self,
+        message: str = "",
+        *,
+        failure_kind: str = "",
+        compare_failures: Optional[List[Dict[str, Any]]] = None,
+        baseline_updated: Optional[List[str]] = None,
+        assertion_results: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        super().__init__(message)
+        self.failure_kind = failure_kind
+        self.compare_failures = compare_failures or []
+        self.baseline_updated = baseline_updated or []
+        self.assertion_results = assertion_results or []
+
+
+def _build_diff_summary(result: Any) -> Dict[str, Any]:
+    """Extract a numerical diff summary from a ``ComparisonResult``.
+
+    Walks the ``differences`` list and computes:
+    - ``total_differences``
+    - ``max_rel_error`` / ``max_abs_error`` (for cells that are numeric)
+    - ``max_rel_error_at`` / ``max_abs_error_at`` (position string)
+    """
+    differences = getattr(result, "differences", []) or []
+    summary: Dict[str, Any] = {
+        "total_differences": len(differences),
+        "max_rel_error": None,
+        "max_rel_error_at": None,
+        "max_abs_error": None,
+        "max_abs_error_at": None,
+    }
+    for diff in differences:
+        exp = getattr(diff, "expected", None)
+        act = getattr(diff, "actual", None)
+        pos = getattr(diff, "position", None)
+        try:
+            ve = float(exp)
+            va = float(act)
+            abs_err = abs(va - ve)
+            rel_err = abs_err / max(abs(ve), 1e-300) if abs(ve) > 0 else float("inf")
+            if summary["max_abs_error"] is None or abs_err > summary["max_abs_error"]:
+                summary["max_abs_error"] = abs_err
+                summary["max_abs_error_at"] = pos
+            if summary["max_rel_error"] is None or rel_err > summary["max_rel_error"]:
+                summary["max_rel_error"] = rel_err
+                summary["max_rel_error_at"] = pos
+        except (ValueError, TypeError):
+            pass
+    return summary
+
+
+class Assertions:
+    @staticmethod
+    def equals(actual: Any, expected: Any, message: str = "") -> bool:
+        if actual != expected:
+            raise AssertionError(f"{message} Expected: {expected}, but got: {actual}")
+        return True
+
+    @staticmethod
+    def contains(container: str, item: str, message: str = "") -> bool:
+        """
+        Check if the item is contained within the container string.
+        This method returns True if the item is found anywhere within the container,
+        even if the container contains other information.
+        """
+        if item not in container:
+            raise AssertionError(f"{message} Expected to contain: {item}")
+        return True
+
+    @staticmethod
+    def matches(text: str, pattern: str, message: str = "") -> bool:
+        if not re.search(pattern, text):
+            raise AssertionError(f"{message} Text does not match pattern: {pattern}")
+        return True
+
+    @staticmethod
+    def return_code_equals(actual: int, expected: int, message: str = "") -> bool:
+        if actual != expected:
+            raise AssertionError(f"{message} Expected return code: {expected}, got: {actual}")
+        return True
+
+    @staticmethod
+    def compare_files(
+        actual_path: str,
+        baseline_path: str,
+        file_type: Optional[str] = None,
+        workspace: Optional[str] = None,
+        *,
+        update_baseline: bool = False,
+        error_analysis: bool = False,
+        **comparator_kwargs: Any,
+    ) -> Dict[str, Any]:
+        """
+        Compare two files using the appropriate file comparator.
+
+        :param actual_path:   Path to the file generated by the test command.
+        :param baseline_path: Path to the golden / reference file.
+        :param file_type:     Comparator type ('h5','json','csv','xml','text','binary').
+                              Auto-detected from file extension if omitted.
+        :param workspace:     Working directory; both paths are resolved relative to
+                              this directory when they are not absolute.
+        :param update_baseline: If True, on comparison failure copy ``actual`` over
+                                ``baseline`` and report the update.
+        :param error_analysis: If True, enable streaming error statistics over ALL
+                               numeric cells (CSV/H5 comparators).
+        :param comparator_kwargs: Extra keyword arguments forwarded to the comparator
+                                  (e.g. ``rtol=1e-5``, ``atol=1e-8``, ``encoding='utf-8'``).
+        :return: A dict with ``identical``, ``error``, ``diff_summary``,
+                 ``differences``, ``actual``, ``baseline``, ``type``,
+                 ``comparator_kwargs``, ``baseline_updated``.
+        :raises ValidationError: when files differ (unless update_baseline is enabled).
+        """
+        # Resolve paths relative to workspace
+        orig_actual = actual_path
+        orig_baseline = baseline_path
+        if actual_path and workspace and not os.path.isabs(actual_path):
+            actual_path = os.path.join(workspace, actual_path)
+        if baseline_path and workspace and not os.path.isabs(baseline_path):
+            baseline_path = os.path.join(workspace, baseline_path)
+
+        # Auto-detect file type from extension
+        if not file_type:
+            if not actual_path:
+                raise ValidationError(
+                    "File type cannot be auto-detected: 'actual' path is empty. "
+                    "Specify 'type' explicitly (e.g. 'script' or a custom plugin type).",
+                    failure_kind="file_compare",
+                )
+            file_type = _detect_file_type(actual_path)
+
+        # Extract compare_files() method-level parameters (not constructor kwargs).
+        # These control line/column ranges in the comparator's compare_files() call.
+        _method_keys = {"start_line", "end_line", "start_column", "end_column"}
+        method_params: Dict[str, Any] = {}
+        for k in list(comparator_kwargs):
+            if k in _method_keys:
+                method_params[k] = comparator_kwargs.pop(k)
+
+        # Convert 1-based user input to 0-based (matches CLI behaviour)
+        if "start_line" in method_params:
+            method_params["start_line"] = max(0, int(method_params["start_line"]) - 1)
+        if "end_line" in method_params and method_params["end_line"] is not None:
+            method_params["end_line"] = max(0, int(method_params["end_line"]) - 1)
+        if "start_column" in method_params:
+            method_params["start_column"] = max(0, int(method_params["start_column"]) - 1)
+        if "end_column" in method_params and method_params["end_column"] is not None:
+            method_params["end_column"] = max(0, int(method_params["end_column"]) - 1)
+
+        # Collect tolerances for reporting (before they're consumed by factory)
+        reported_kwargs = {
+            k: v for k, v in comparator_kwargs.items()
+            if k not in ("verbose", "debug", "num_threads", "chunk_size", "encoding")
+        }
+
+        try:
+            comparator = ComparatorFactory.create_comparator(
+                file_type,
+                verbose=True,  # always include diff details in the assertion message
+                error_analysis=error_analysis,
+                **comparator_kwargs,
+            )
+            result = comparator.compare_files(actual_path, baseline_path, **method_params)
+
+            # Build structured response
+            diff_summary = _build_diff_summary(result)
+            response: Dict[str, Any] = {
+                "identical": result.identical,
+                "error": result.error,
+                "actual": orig_actual,
+                "baseline": orig_baseline,
+                "type": file_type,
+                "comparator_kwargs": reported_kwargs,
+                "diff_summary": diff_summary,
+                "differences": (
+                    [d.to_dict() for d in result.differences]
+                    if result.differences else []
+                ),
+                "error_stats": result.error_stats,
+                "command_output": result.command_output,
+                "baseline_updated": False,
+            }
+
+            if result.error:
+                raise ValidationError(
+                    f"File comparison error ({orig_actual} vs {orig_baseline}): {result.error}",
+                    failure_kind="file_compare",
+                    compare_failures=[response],
+                )
+
+            if not result.identical:
+                if update_baseline:
+                    # Overwrite baseline with actual
+                    os.makedirs(os.path.dirname(baseline_path), exist_ok=True)
+                    shutil.copy2(actual_path, baseline_path)
+                    response["baseline_updated"] = True
+                    response["identical"] = True  # treated as pass
+                    logger.info(
+                        "  [UPDATE BASELINE] %s → %s",
+                        orig_actual, orig_baseline,
+                    )
+                    return response
+                else:
+                    raise ValidationError(
+                        f"File comparison failed ({orig_actual} vs {orig_baseline}):\n{result}",
+                        failure_kind="file_compare",
+                        compare_failures=[response],
+                    )
+
+            return response
+
+        except ValidationError:
+            raise
+        except Exception as exc:
+            raise ValidationError(
+                f"File comparison error ({orig_actual} vs {orig_baseline}): {exc}",
+                failure_kind="file_compare",
+            )

symtest/core/base_runner.py

@@ -0,0 +1,299 @@
+import time
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from .test_case import TestCase
+from .assertions import Assertions
+from .setup import SetupManager, EnvironmentSetup
+from .execution import execute_single_test_case
+from .history_store import load_history, update_case, check_regression, save_history, reset_cases
+from .last_run_store import update_last_run, get_last_failed_names
+from ..file_comparator.factory import ComparatorFactory
+
+logger = logging.getLogger("symtest.core.base_runner")
+
+class BaseRunner(ABC):
+    def __init__(self, config_file: str, workspace: Optional[str] = None,
+                 test_case_filter: Optional[List[str]] = None,
+                 test_case_tag_filter: Optional[List[str]] = None,
+                 history_dir: Optional[str] = None,
+                 regression_threshold: float = 1.5,
+                 update_baseline: bool = False,
+                 update_history: bool = False,
+                 error_analysis: bool = False,
+                 last_failed: bool = False,
+                 resume: bool = False,
+                 plugin_dirs: Optional[List[str]] = None):
+        if workspace:
+            self.workspace = Path(workspace)
+        else:
+            self.workspace = Path.cwd()
+        config_path = Path(config_file)
+        if config_path.is_absolute():
+            self.config_path = config_path
+        else:
+            self.config_path = self.workspace / config_path
+        self.test_cases: List[TestCase] = []
+        self.test_case_filter: Optional[List[str]] = test_case_filter
+        self.test_case_tag_filter: Optional[List[str]] = test_case_tag_filter
+        if history_dir:
+            self.history_dir = str((self.workspace / history_dir).resolve())
+        else:
+            self.history_dir = None
+        self.regression_threshold = regression_threshold
+        self.update_baseline = update_baseline
+        self.update_history = update_history
+        self.error_analysis = error_analysis
+        self.last_failed = last_failed
+        self.resume = resume
+
+        # --- workspace plugin directories ---
+        resolved_plugin_dirs: List[str] = list(plugin_dirs) if plugin_dirs else []
+        default_plugin_dir = self.workspace / "comparators"
+        if default_plugin_dir.is_dir() and str(default_plugin_dir.resolve()) not in resolved_plugin_dirs:
+            resolved_plugin_dirs.append(str(default_plugin_dir.resolve()))
+        ComparatorFactory.set_plugin_dirs(resolved_plugin_dirs)
+        self.results: Dict[str, Any] = {
+            "total": 0,
+            "passed": 0,
+            "failed": 0,
+            "xfailed": 0,
+            "xpassed": 0,
+            "updated": 0,
+            "details": []
+        }
+        self.assertions = Assertions()
+        self.setup_manager = SetupManager()
+
+    @abstractmethod
+    def load_test_cases(self) -> None:
+        """Load test cases from configuration file"""
+        pass
+    
+    def load_setup_from_config(self, config: Dict[str, Any]) -> None:
+        """从配置文件加载setup配置"""
+        setup_config = config.get("setup", {})
+        
+        # 处理环境变量设置
+        if "environment_variables" in setup_config:
+            env_setup = EnvironmentSetup({"environment_variables": setup_config["environment_variables"]})
+            self.setup_manager.add_setup(env_setup)
+        
+        # 这里可以扩展支持其他类型的setup插件
+        # 例如：
+        # if "custom_setups" in setup_config:
+        #     for custom_setup_config in setup_config["custom_setups"]:
+        #         # 动态加载自定义setup插件
+        #         pass
+
+    def _apply_test_case_filter(self) -> None:
+        """根据 test_case_filter / test_case_tag_filter / --last-failed 过滤测试用例"""
+        if self.last_failed and not self.test_case_filter:
+            ws = str(self.workspace) if self.workspace else str(Path.cwd())
+            failed_names = get_last_failed_names(ws)
+            if failed_names:
+                logger.info(
+                    "--last-failed: filtering to %d previously failed case(s): %s",
+                    len(failed_names), ", ".join(failed_names),
+                )
+                self.test_case_filter = (
+                    (self.test_case_filter or []) + failed_names
+                )
+            else:
+                logger.info("--last-failed: no previously failed cases found; running all.")
+
+        if self.test_case_filter or self.test_case_tag_filter:
+            original_count = len(self.test_cases)
+            self.test_cases = [
+                tc for tc in self.test_cases
+                if (not self.test_case_filter or tc.name in self.test_case_filter)
+                and (not self.test_case_tag_filter
+                     or set(tc.tags or []) & set(self.test_case_tag_filter))
+            ]
+            filtered_out = original_count - len(self.test_cases)
+            if filtered_out > 0:
+                logger.info("Filtered out %d test case(s). Running %d specified case(s).",
+                            filtered_out, len(self.test_cases))
+            if not self.test_cases:
+                logger.warning("No matching test cases found for: names=%s, tags=%s",
+                               self.test_case_filter, self.test_case_tag_filter)
+
+    def run_tests(self) -> bool:
+        """Run all test cases and return whether all tests passed"""
+        try:
+            self.load_test_cases()
+            self._apply_test_case_filter()
+            self.results["total"] = len(self.test_cases)
+            
+            if self.results["total"] == 0:
+                logger.warning("No test cases to run.")
+                return False
+            
+            # 执行setup任务
+            self.setup_manager.setup_all()
+            
+            total_start_time = time.time()
+            
+            logger.info("Starting test execution... Total tests: %d", self.results["total"])
+            logger.info("=" * 50)
+            
+            for i, case in enumerate(self.test_cases, 1):
+                logger.info("Running test %d/%d: %s", i, self.results["total"], case.name)
+                result = self.run_single_test(case)
+                
+                # Apply xfail status mapping before counting
+                self._apply_xfail_status(result, case)
+
+                # ── Echo expected / description / tags ──
+                result["expected"] = case.expected if case.expected else None
+                result["description"] = case.description or None
+                result["tags"] = case.tags or []
+                self._fill_hint_command(result, case.name)
+
+                self.results["details"].append(result)
+                duration = result.get("duration", 0)
+                status = result["status"]
+                if status == "passed":
+                    self.results["passed"] += 1
+                    # Check for baseline updates
+                    if result.get("baseline_updated"):
+                        self.results["updated"] += 1
+                        logger.info("✓ Test passed (baseline updated): %s (%.2fs)", case.name, duration)
+                    elif result.get("flaky"):
+                        logger.info("✓ Test passed (flaky, %d attempts): %s (%.2fs)",
+                                    result.get("attempts", 1), case.name, duration)
+                    else:
+                        logger.info("✓ Test passed: %s (%.2fs)", case.name, duration)
+                elif status == "xfailed":
+                    self.results["xfailed"] += 1
+                    attempt_info = f" ({result.get('attempts', 1)} attempts)" if result.get("attempts", 1) > 1 else ""
+                    logger.info("✓ Test xfailed (expected)%s: %s (%.2fs)", attempt_info, case.name, duration)
+                    if result.get("message"):
+                        logger.info("  Reason: %s", result.get("xfail_reason", ""))
+                        logger.info("  Detail: %s", result["message"])
+                elif status == "xpassed":
+                    self.results["xpassed"] += 1
+                    self.results["failed"] += 1
+                    logger.error("✗ Test xpassed (unexpected!): %s (%.2fs)", case.name, duration)
+                    if result.get("message"):
+                        logger.error("  Error: %s", result["message"])
+                    logger.warning("  [XPass] Marked as expected_failure but passed — remove the xfail marker.")
+                else:
+                    self.results["failed"] += 1
+                    if result.get("flaky"):
+                        logger.error("✗ Test failed (%d attempts): %s (%.2fs)",
+                                     result.get("attempts", 1), case.name, duration)
+                    else:
+                        logger.error("✗ Test failed: %s (%.2fs)", case.name, duration)
+                    if result["message"]:
+                        logger.error("  Error: %s", result["message"])
+                    
+            total_duration = time.time() - total_start_time
+            logger.info("=" * 50)
+            logger.info(
+                "Test execution completed in %.2fs. "
+                "Passed: %d, Failed: %d, XFailed: %d, XPassed: %d",
+                total_duration,
+                self.results["passed"], self.results["failed"],
+                self.results["xfailed"], self.results["xpassed"],
+            )
+
+            # Update history & regression detection
+            self._update_history()
+
+            # Save last-run state for --last-failed
+            self._save_last_run()
+
+            # Exit-code rule: failed + xpassed > 0 → non-zero
+            return self.results["failed"] == 0 and self.results["xpassed"] == 0
+        finally:
+            # 确保teardown总是被执行
+            self.setup_manager.teardown_all()
+
+    def _fill_hint_command(self, result: Dict[str, Any], case_name: str) -> None:
+        """Fill in the concrete CLI command inside ``next_action_hint``.
+
+        The execution layer attaches the hint with ``command=None`` because it
+        does not know the config file path; the runner does.
+        """
+        hint = result.get("next_action_hint")
+        if not hint or hint.get("command"):
+            return
+        config = str(self.config_path)
+        if hint.get("action") == "update_baseline":
+            hint["command"] = (
+                f'symtest run "{config}" --update-baseline -t "{case_name}"'
+            )
+        else:
+            hint["command"] = f'symtest run "{config}" -t "{case_name}"'
+
+    def _apply_xfail_status(self, result: Dict[str, Any], case: "TestCase") -> None:
+        """Apply xfail (expected failure) status mapping to a test result.
+
+        When ``case.expected_failure`` is True:
+        - ``passed`` → ``xpassed`` (unexpected pass; counts as a suite failure)
+        - any non-passed status → ``xfailed`` (expected failure; not a failure)
+
+        The ``xfail_reason`` from the case is attached to the result dict so the
+        report can display it.
+        """
+        if not getattr(case, "expected_failure", False):
+            return
+        xfail_reason = getattr(case, "xfail_reason", "") or ""
+        result["xfail_reason"] = xfail_reason
+        result["xfail_quiet"] = getattr(case, "xfail_quiet", False)
+        if result.get("status") == "passed":
+            result["status"] = "xpassed"
+        else:
+            result["status"] = "xfailed"
+
+    def _save_last_run(self) -> None:
+        """Persist per-case status for ``--last-failed`` support."""
+        ws = str(self.workspace) if self.workspace else str(Path.cwd())
+        update_last_run(ws, self.results["details"])
+
+    def _update_history(self) -> None:
+        """Update .symtest history with successful run results and check for regressions."""
+        if not self.history_dir:
+            return
+        history = load_history(self.history_dir)
+
+        if self.update_history:
+            run_names = {r["name"] for r in self.results["details"]}
+            cleared = reset_cases(history, run_names)
+            if cleared:
+                logger.info(
+                    "History reset: cleared %d case(s) before recording this run",
+                    cleared,
+                )
+                self.results["history_reset"] = True
+                self.results["history_cleared"] = cleared
+
+        for result in self.results["details"]:
+            # Only record successful cases in history; skip failed ones
+            if result["status"] != "passed":
+                continue
+            duration = result.get("duration", 0)
+            # Check regression BEFORE updating (compare against old avg)
+            warning = check_regression(history, result["name"], duration, self.regression_threshold)
+            if warning:
+                logger.warning(warning)
+            update_case(history, result["name"], duration)
+        save_history(self.history_dir, history)
+
+    def _run_sequence(self, case: TestCase) -> Dict[str, Any]:
+        """Run a sequence test case with multiple steps (fail-fast)."""
+        from .config_loader import execute_sequence
+        return execute_sequence(
+            case_name=case.name,
+            steps=case.steps,
+            workspace=str(self.workspace) if self.workspace else None,
+            case_expected=case.expected if case.expected else None,
+            resume=self.resume,
+        )
+
+    @abstractmethod
+    def run_single_test(self, case: TestCase) -> Dict[str, str]:
+        """Run a single test case and return the result"""
+        pass