symtest-cli 1.3.0__py3-none-any.whl

symtest/core/last_run_store.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+
+"""
+Last-run state store for ``--last-failed`` support.
+
+Stores per-case status in ``<workspace>/.cli-test/last_run.json``.
+Each run **overwrites** the status of every case that was executed,
+so "previously failed but now fixed" cases are immediately removed
+from the failed set.  Cases that were not executed in this run
+retain their previous status.
+"""
+
+import json
+import os
+from typing import Dict, List, Optional
+
+LAST_RUN_DIR = ".cli-test"
+LAST_RUN_FILENAME = "last_run.json"
+
+
+def _last_run_path(workspace: str) -> str:
+    """Return the full path to ``last_run.json`` for a workspace."""
+    return os.path.join(workspace, LAST_RUN_DIR, LAST_RUN_FILENAME)
+
+
+def load_last_run(workspace: str) -> Dict:
+    """Load last-run state. Returns empty dict when file doesn't exist."""
+    path = _last_run_path(workspace)
+    if not os.path.exists(path):
+        return {}
+    with open(path, "r", encoding="utf-8") as f:
+        try:
+            return json.load(f)
+        except (json.JSONDecodeError, ValueError):
+            return {}
+
+
+def save_last_run(workspace: str, data: Dict) -> None:
+    """Write last-run state, creating the directory if needed."""
+    dir_path = os.path.join(workspace, LAST_RUN_DIR)
+    os.makedirs(dir_path, exist_ok=True)
+    path = _last_run_path(workspace)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+
+def update_last_run(
+    workspace: str,
+    results: List[Dict],
+) -> None:
+    """Update last-run state with the results from the current execution.
+
+    For each result dict in *results*:
+    - The case's entry is **overwritten** with the current status.
+    - Cases in the existing file that were NOT part of this run
+      retain their previous status (e.g. when running a subset via ``-t``).
+
+    This ensures ``--last-failed`` never includes a case that passed
+    in the most recent run, since passing results overwrite any prior
+    failure record.
+    """
+    data = load_last_run(workspace)
+
+    for result in results:
+        name = result.get("name", "")
+        if not name:
+            continue
+        # Only store status + the case name
+        data[name] = {
+            "status": result.get("status", "unknown"),
+        }
+
+    save_last_run(workspace, data)
+
+
+def get_last_failed_names(workspace: str) -> List[str]:
+    """Return the names of cases that failed in the last run.
+
+    Returns an empty list when no last-run state exists (first run).
+    """
+    data = load_last_run(workspace)
+    if not data:
+        return []
+    return sorted(
+        name for name, info in data.items()
+        if info.get("status") in ("failed", "timeout", "xpassed")
+    )
+
+
+def get_last_run_summary(workspace: str) -> Dict[str, int]:
+    """Return a summary counts dict: total, passed, failed, xfailed, xpassed, timeout."""
+    data = load_last_run(workspace)
+    summary = {"total": 0, "passed": 0, "failed": 0, "xfailed": 0, "xpassed": 0, "timeout": 0}
+    for info in data.values():
+        st = info.get("status", "unknown")
+        summary["total"] += 1
+        if st == "passed":
+            summary["passed"] += 1
+        elif st == "xfailed":
+            summary["xfailed"] += 1
+        elif st == "xpassed":
+            summary["xpassed"] += 1
+            summary["failed"] += 1  # xpassed IS a suite failure
+        elif st == "failed":
+            summary["failed"] += 1
+        elif st == "timeout":
+            summary["timeout"] += 1
+            summary["failed"] += 1  # timeout counts in failure aggregate
+    return summary

symtest/core/parallel_runner.py

@@ -0,0 +1,251 @@
+from abc import ABC
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed
+from typing import List, Dict, Any, Optional, Union
+import time
+import threading
+import logging
+from .base_runner import BaseRunner
+from .test_case import TestCase
+from .process_worker import run_test_in_process
+
+logger = logging.getLogger("symtest.core.parallel_runner")
+
+
+class AtomicSemaphore:
+    """
+    支持原子级多令牌获取的信号量，消除逐个 acquire 导致的部分占有死锁。
+
+    与 threading.Semaphore 不同：acquire(n) 在所有 n 个令牌可用时一次性获取，
+    否则等待（支持超时），不会出现"拿了3个等1个，另一个线程也拿了3个等1个"的死锁。
+
+    唤醒策略：按请求令牌数降序优先唤醒，避免大核数任务被小任务持续抢占导致饥饿。
+    """
+
+    def __init__(self, value: int):
+        self._value = value
+        self._lock = threading.Lock()
+        self._waiters: list = []  # list of (required_n, threading.Event)
+
+    def _grant_tokens(self) -> None:
+        """Grant tokens to eligible waiters, largest request first (anti-starvation)."""
+        if not self._waiters:
+            return
+        # Sort by required_n descending so large-core requests get priority
+        self._waiters.sort(key=lambda x: -x[0])
+        granted: list = []
+        remaining: list = []
+        for n, event in self._waiters:
+            if self._value >= n:
+                self._value -= n
+                granted.append(event)
+            else:
+                remaining.append((n, event))
+        self._waiters = remaining
+        for event in granted:
+            event.set()
+
+    def acquire(self, n: int = 1, timeout: Optional[float] = None) -> bool:
+        """Atomically acquire n tokens. Returns True on success, False on timeout."""
+        event = threading.Event()
+        with self._lock:
+            # Fast path: enough tokens and no pending waiters
+            if self._value >= n and not self._waiters:
+                self._value -= n
+                return True
+            self._waiters.append((n, event))
+            self._grant_tokens()
+
+        if not event.wait(timeout=timeout):
+            # Timeout cleanup – guard against race where release granted
+            # tokens between event.wait() returning and lock acquisition
+            with self._lock:
+                if event.is_set():
+                    return True
+                for i, (_, e) in enumerate(self._waiters):
+                    if e is event:
+                        del self._waiters[i]
+                        self._grant_tokens()
+                        break
+            return False
+        return True
+
+    def release(self, n: int = 1) -> None:
+        """Release n tokens, waking eligible waiters with anti-starvation priority."""
+        with self._lock:
+            self._value += n
+            self._grant_tokens()
+
+class ParallelRunner(BaseRunner):
+    """并行测试运行器基类，支持多线程和多进程执行"""
+    
+    def __init__(self, config_file: str, workspace: Optional[str] = None, 
+                 max_workers: Optional[int] = None, 
+                 execution_mode: str = "thread",
+                 error_analysis: bool = False,
+                 **kwargs):
+        """
+        初始化并行运行器
+        
+        Args:
+            config_file: 配置文件路径
+            workspace: 工作目录
+            max_workers: 最大并发数，默认为CPU核心数
+            execution_mode: 执行模式，'thread'(线程) 或 'process'(进程)
+            **kwargs: 透传给 BaseRunner 的额外参数
+                (test_case_filter, test_case_tag_filter, history_dir, regression_threshold)
+        """
+        super().__init__(config_file, workspace, **kwargs)
+        self.max_workers = max_workers
+        self.execution_mode = execution_mode
+        self.error_analysis = error_analysis
+        self.lock = threading.Lock()  # 用于线程安全的结果更新
+        
+    def run_tests(self) -> bool:
+        """并行运行所有测试用例"""
+        try:
+            self.load_test_cases()
+            self._apply_test_case_filter()
+            self.results["total"] = len(self.test_cases)
+            
+            if self.results["total"] == 0:
+                logger.warning("No test cases to run.")
+                return False
+            
+            # 执行setup任务
+            self.setup_manager.setup_all()
+            
+            logger.info("Starting parallel test execution... Total tests: %d", self.results["total"])
+            logger.info("Execution mode: %s, Max workers: %s", self.execution_mode, self.max_workers or "auto")
+            logger.info("=" * 50)
+            
+            start_time = time.time()
+            
+            if self.execution_mode == "process":
+                executor_class = ProcessPoolExecutor
+            else:
+                executor_class = ThreadPoolExecutor
+                
+            with executor_class(max_workers=self.max_workers) as executor:
+                # 提交所有测试任务
+                if self.execution_mode == "process":
+                    # 进程模式：使用独立的工作器函数
+                    future_to_case = {
+                        executor.submit(
+                            run_test_in_process, 
+                            i, 
+                            {
+                                "name": case.name,
+                                "command": case.command,
+                                "args": case.args,
+                                "expected": case.expected,
+                                "timeout": case.timeout,
+                                "resources": case.resources,
+                                "retry_count": case.retry_count,
+                                "steps": [
+                                    {
+                                        "command": s.command,
+                                        "args": s.args,
+                                        "expected": s.expected,
+                                        "timeout": s.timeout,
+                                        "retry_count": s.retry_count,
+                                    }
+                                    for s in case.steps
+                                ] if case.steps else None,
+                            },
+                            str(self.workspace) if self.workspace else None,
+                            update_baseline=self.update_baseline,
+                            error_analysis=self.error_analysis,
+                            resume=self.resume,
+                        ): (i, case) 
+                        for i, case in enumerate(self.test_cases, 1)
+                    }
+                else:
+                    # 线程模式：使用实例方法
+                    future_to_case = {
+                        executor.submit(self._run_test_with_index, i, case): (i, case) 
+                        for i, case in enumerate(self.test_cases, 1)
+                    }
+                
+                # 收集结果
+                for future in as_completed(future_to_case):
+                    test_index, case = future_to_case[future]
+                    try:
+                        result = future.result()
+                        self._update_results(result, test_index, case)
+                    except Exception as exc:
+                        error_result = {
+                            "name": case.name,
+                            "status": "failed",
+                            "message": f"Test execution failed: {str(exc)}",
+                            "output": "",
+                            "command": "",
+                            "return_code": None
+                        }
+                        self._update_results(error_result, test_index, case)
+            
+            end_time = time.time()
+            execution_time = end_time - start_time
+            
+            logger.info("=" * 50)
+            logger.info(
+                "Parallel test execution completed in %.2f seconds", execution_time,
+            )
+            logger.info(
+                "Passed: %d, Failed: %d, XFailed: %d, XPassed: %d",
+                self.results["passed"], self.results["failed"],
+                self.results["xfailed"], self.results["xpassed"],
+            )
+
+            # Update history & regression detection
+            self._update_history()
+
+            # Save last-run state for --last-failed
+            self._save_last_run()
+
+            # Exit-code rule: failed + xpassed > 0 → non-zero
+            return self.results["failed"] == 0 and self.results["xpassed"] == 0
+        finally:
+            # 确保teardown总是被执行
+            self.setup_manager.teardown_all()
+    
+    def _run_test_with_index(self, test_index: int, case: TestCase) -> Dict[str, Any]:
+        """运行单个测试并返回结果（包含索引信息）"""
+        logger.info("[Worker] Running test %d: %s", test_index, case.name)
+        result = self.run_single_test(case)
+        return result
+    
+    def _update_results(self, result: Dict[str, Any], test_index: int, case: TestCase) -> None:
+        """线程安全地更新测试结果"""
+        with self.lock:
+            # Apply xfail status mapping before counting
+            self._apply_xfail_status(result, case)
+
+            self._fill_hint_command(result, case.name)
+            self.results["details"].append(result)
+            duration = result.get("duration", 0)
+            status = result["status"]
+            if status == "passed":
+                self.results["passed"] += 1
+                logger.info("✓ Test %d passed: %s (%.2fs)", test_index, case.name, duration)
+            elif status == "xfailed":
+                self.results["xfailed"] += 1
+                logger.info("✓ Test %d xfailed (expected): %s (%.2fs)", test_index, case.name, duration)
+                if result.get("message"):
+                    logger.info("  Detail: %s", result["message"])
+            elif status == "xpassed":
+                self.results["xpassed"] += 1
+                self.results["failed"] += 1
+                logger.error("✗ Test %d xpassed (unexpected!): %s (%.2fs)", test_index, case.name, duration)
+                if result.get("message"):
+                    logger.error("  Error: %s", result["message"])
+                logger.warning("  [XPass] Marked as expected_failure but passed — remove the xfail marker.")
+            else:
+                self.results["failed"] += 1
+                logger.error("✗ Test %d failed: %s (%.2fs)", test_index, case.name, duration)
+                if result["message"]:
+                    logger.error("  Error: %s", result["message"])
+    
+    def run_tests_sequential(self) -> bool:
+        """回退到顺序执行模式"""
+        logger.info("Falling back to sequential execution...")
+        return super().run_tests() 

symtest/core/process_worker.py

@@ -0,0 +1,93 @@
+"""
+进程工作器模块
+用于多进程并行测试执行，避免序列化问题
+"""
+
+import logging
+from typing import Dict, Any, List
+from .config_loader import execute_sequence
+from .execution import execute_single_test_case
+from .types import TestCaseData
+
+logger = logging.getLogger("symtest.core.process_worker")
+
+def _run_sequence_in_process(
+    test_index: int,
+    case_data: Dict[str, Any],
+    workspace: str = None,
+    *,
+    update_baseline: bool = False,
+    error_analysis: bool = False,
+    resume: bool = False,
+) -> Dict[str, Any]:
+    """Run a sequence test case with multiple steps (fail-fast) in a process worker."""
+    return execute_sequence(
+        case_name=case_data["name"],
+        steps=case_data["steps"],
+        workspace=workspace,
+        print_prefix=f"[Process Worker {test_index}]",
+        executor=execute_single_test_case,
+        case_expected=case_data.get("expected"),
+        update_baseline=update_baseline,
+        error_analysis=error_analysis,
+        resume=resume,
+    )
+
+
+def run_test_in_process(
+    test_index: int,
+    case_data: Dict[str, Any],
+    workspace: str = None,
+    *,
+    update_baseline: bool = False,
+    error_analysis: bool = False,
+    resume: bool = False,
+) -> Dict[str, Any]:
+    """
+    在独立进程中运行单个测试用例
+    
+    Args:
+        test_index: 测试索引
+        case_data: 测试用例数据字典
+        workspace: 工作目录
+        update_baseline: 是否更新基线文件
+        resume: 是否启用断点续跑
+    
+    Returns:
+        测试结果字典
+    """
+    # Sequence mode
+    if case_data.get("steps"):
+        return _run_sequence_in_process(
+            test_index, case_data, workspace,
+            update_baseline=update_baseline,
+            error_analysis=error_analysis,
+            resume=resume,
+        )
+
+    # Single command mode
+    case: TestCaseData = {
+        "name": case_data["name"],
+        "command": case_data["command"],
+        "args": case_data["args"],
+        "expected": case_data["expected"],
+        "description": case_data.get("description"),
+        "timeout": case_data.get("timeout"),
+        "resources": case_data.get("resources"),
+        "retry_count": case_data.get("retry_count", 0),
+    }
+
+    command_preview = f"{case['command']} {' '.join(case['args'])}".strip()
+    logger.info("  [Process Worker %d] Executing command: %s", test_index, command_preview)
+
+    result = execute_single_test_case(case, workspace, update_baseline=update_baseline, error_analysis=error_analysis)
+
+    if result["output"].strip():
+        logger.debug("  [Process Worker %d] Command output for %s:", test_index, case["name"])
+        for line in result["output"].splitlines():
+            logger.debug("    %s", line)
+
+    if result["status"] != "passed" and result.get("message"):
+        logger.error("  [Process Worker %d] Error for %s: %s", test_index, case["name"], result["message"])
+
+    return result 

symtest/core/sequence_state.py

@@ -0,0 +1,143 @@
+"""
+Sequence state store for step-level resume (``--resume``).
+
+Stores intermediate step results so that when a sequence test case fails,
+re-running with ``--resume`` skips already-passed steps and reconstructs
+``combined_output`` from cached step outputs.
+
+State file: ``<workspace>/.cli-test/sequence_state/<case_name>.json``
+Output cache: ``<workspace>/.cli-test/sequence_state/cache/<case_name>.step<N>.log``
+
+**Trust model**: ``--resume`` trusts that workspace artifacts (input files,
+pre-step outputs) have not been modified between runs.  No artifact
+validation is performed — the user asserts correctness by opting into resume.
+"""
+
+import hashlib
+import json
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger("symtest.core.sequence_state")
+
+SEQUENCE_STATE_DIR = ".cli-test/sequence_state"
+CACHE_SUBDIR = "cache"
+
+
+def _state_dir_path(workspace: str) -> str:
+    return os.path.join(workspace, SEQUENCE_STATE_DIR)
+
+
+def _cache_dir_path(workspace: str) -> str:
+    return os.path.join(_state_dir_path(workspace), CACHE_SUBDIR)
+
+
+def _case_state_path(workspace: str, case_name: str) -> str:
+    return os.path.join(_state_dir_path(workspace), f"{case_name}.json")
+
+
+def _step_output_path(workspace: str, case_name: str, step_idx: int) -> str:
+    dir_path = _cache_dir_path(workspace)
+    os.makedirs(dir_path, exist_ok=True)
+    return os.path.join(dir_path, f"{case_name}.step{step_idx}.log")
+
+
+def compute_config_hash(
+    steps: List[Any],
+    case_expected: Optional[Dict[str, Any]] = None,
+) -> str:
+    """Compute a deterministic SHA-256 hash of the step configuration.
+
+    If any step's command/args/expected or the case-level expected block
+    changes, the hash will differ and ``--resume`` will fall back to a full
+    re-run.
+    """
+    def field(step: Any, name: str, default: Any) -> Any:
+        if hasattr(step, name):
+            return getattr(step, name)
+        return step.get(name, default)
+
+    payload = {
+        "steps": [
+            {
+                "command": field(step, "command", ""),
+                # Argument order is significant for command-line programs.
+                "args": field(step, "args", []),
+                "expected": field(step, "expected", {}),
+                "timeout": field(step, "timeout", None),
+                "retry_count": field(step, "retry_count", 0),
+            }
+            for step in steps
+        ],
+        "case_expected": case_expected or {},
+    }
+    canonical = json.dumps(
+        payload,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def load_sequence_state(
+    workspace: str, case_name: str
+) -> Optional[Dict[str, Any]]:
+    """Load persisted step state for a case.  Returns ``None`` if not found."""
+    path = _case_state_path(workspace, case_name)
+    if not os.path.exists(path):
+        return None
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, ValueError):
+        logger.warning("Corrupted sequence state for %s, ignoring.", case_name)
+        return None
+
+
+def save_sequence_state(
+    workspace: str, case_name: str, state: Dict[str, Any]
+) -> None:
+    """Persist step state for a case."""
+    dir_path = _state_dir_path(workspace)
+    os.makedirs(dir_path, exist_ok=True)
+    path = _case_state_path(workspace, case_name)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(state, f, indent=2, ensure_ascii=False)
+
+
+def delete_sequence_state(workspace: str, case_name: str) -> None:
+    """Remove state file and cached step outputs (case fully passed)."""
+    # Remove state file
+    path = _case_state_path(workspace, case_name)
+    if os.path.exists(path):
+        os.remove(path)
+    # Remove cached step outputs
+    for step_idx in range(1, 100):  # reasonable upper bound
+        out_path = _step_output_path(workspace, case_name, step_idx)
+        if os.path.exists(out_path):
+            os.remove(out_path)
+        else:
+            break
+
+
+def save_step_output(
+    workspace: str, case_name: str, step_idx: int, output: str
+) -> str:
+    """Save step output to cache and return the cache file path."""
+    path = _step_output_path(workspace, case_name, step_idx)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(output)
+    return path
+
+
+def load_step_output(
+    workspace: str, case_name: str, step_idx: int
+) -> Optional[str]:
+    """Load cached step output.  Returns ``None`` if not found."""
+    path = _step_output_path(workspace, case_name, step_idx)
+    if not os.path.exists(path):
+        return None
+    with open(path, "r", encoding="utf-8") as f:
+        return f.read()