symtest-cli 1.3.0__py3-none-any.whl

symtest/core/setup.py

@@ -0,0 +1,137 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any, Optional
+import logging
+import os
+
+logger = logging.getLogger("symtest.core.setup")
+
+
+class BaseSetup(ABC):
+    """测试前置任务的基类，允许用户以插件形式自定义"""
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        初始化Setup
+        
+        Args:
+            config: 配置参数字典
+        """
+        self.config = config or {}
+    
+    @abstractmethod
+    def setup(self) -> None:
+        """
+        执行前置任务设置
+        子类必须实现此方法
+        """
+        pass
+    
+    @abstractmethod
+    def teardown(self) -> None:
+        """
+        执行后置清理任务
+        子类必须实现此方法
+        """
+        pass
+    
+    def get_name(self) -> str:
+        """返回Setup的名称"""
+        return self.__class__.__name__
+
+
+class EnvironmentSetup(BaseSetup):
+    """内置的环境变量设置插件"""
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self._original_env = {}  # 存储原始环境变量值
+        self._added_env = set()  # 存储新添加的环境变量
+    
+    def setup(self) -> None:
+        """设置环境变量"""
+        env_vars = self.config.get('environment_variables', {})
+        
+        if not env_vars:
+            return
+            
+        logger.info("Setting up environment variables...")
+        
+        for key, value in env_vars.items():
+            # 保存原始值
+            if key in os.environ:
+                self._original_env[key] = os.environ[key]
+            else:
+                self._added_env.add(key)
+            
+            # 设置新值
+            os.environ[key] = str(value)
+            logger.info("  %s = %s", key, value)
+    
+    def teardown(self) -> None:
+        """恢复环境变量"""
+        if not self._original_env and not self._added_env:
+            return
+            
+        logger.info("Restoring environment variables...")
+        
+        # 恢复原始值
+        for key, value in self._original_env.items():
+            os.environ[key] = value
+            logger.info("  Restored %s", key)
+        
+        # 删除新添加的环境变量
+        for key in self._added_env:
+            if key in os.environ:
+                del os.environ[key]
+                logger.info("  Removed %s", key)
+        
+        # 清空记录
+        self._original_env.clear()
+        self._added_env.clear()
+
+
+class SetupManager:
+    """Setup管理器，负责管理多个Setup插件"""
+    
+    def __init__(self):
+        self.setups = []
+    
+    def add_setup(self, setup: BaseSetup) -> None:
+        """添加Setup插件"""
+        self.setups.append(setup)
+    
+    def setup_all(self) -> None:
+        """执行所有Setup的前置任务"""
+        if not self.setups:
+            return
+            
+        logger.info("=" * 50)
+        logger.info("Executing setup tasks...")
+        logger.info("=" * 50)
+        
+        for setup in self.setups:
+            try:
+                logger.info("Running setup: %s", setup.get_name())
+                setup.setup()
+            except Exception as e:
+                logger.error("Error in setup %s: %s", setup.get_name(), e)
+                raise
+    
+    def teardown_all(self) -> None:
+        """执行所有Setup的后置清理任务（逆序执行）"""
+        if not self.setups:
+            return
+            
+        logger.info("=" * 50)
+        logger.info("Executing teardown tasks...")
+        logger.info("=" * 50)
+        
+        # 逆序执行teardown
+        for setup in reversed(self.setups):
+            try:
+                logger.info("Running teardown: %s", setup.get_name())
+                setup.teardown()
+            except Exception as e:
+                logger.error("Error in teardown %s: %s", setup.get_name(), e)
+                # teardown错误不应该阻止其他teardown的执行
+                continue 

symtest/core/test_case.py

@@ -0,0 +1,76 @@
+from dataclasses import dataclass, field
+from typing import List, Dict, Any, Optional
+
+@dataclass
+class TestCaseStep:
+    """A single step within a sequence test case."""
+    __test__ = False
+    command: str
+    args: List[str]
+    expected: Dict[str, Any]
+    timeout: Optional[float] = None
+    retry_count: int = 0
+
+@dataclass
+class TestCase:
+    __test__ = False
+    name: str
+    command: str = ""
+    args: List[str] = field(default_factory=list)
+    expected: Dict[str, Any] = field(default_factory=dict)
+    description: str = ""
+    timeout: Optional[float] = None
+    resources: Optional[Dict[str, Any]] = None
+    steps: Optional[List[TestCaseStep]] = None
+    tags: List[str] = field(default_factory=list)
+    retry_count: int = 0
+    expected_failure: bool = False
+    xfail_reason: str = ""
+    xfail_quiet: bool = False
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert test case to dictionary format"""
+        result = {
+            "name": self.name,
+            "command": self.command,
+            "args": self.args,
+            "expected": self.expected,
+            "timeout": self.timeout,
+            "resources": self.resources,
+            "tags": self.tags,
+            "retry_count": self.retry_count,
+        }
+        if self.expected_failure:
+            result["expected_failure"] = self.expected_failure
+            result["xfail_reason"] = self.xfail_reason
+            if self.xfail_quiet:
+                result["xfail_quiet"] = self.xfail_quiet
+        if self.steps is not None:
+            result["steps"] = [
+                {
+                    "command": s.command,
+                    "args": s.args,
+                    "expected": s.expected,
+                    "timeout": s.timeout,
+                    "retry_count": s.retry_count,
+                }
+                for s in self.steps
+            ]
+        return result
+
+    def to_execution_dict(self) -> Dict[str, Any]:
+        """Convert to the dict format expected by ``execute_single_test_case``.
+
+        Only for single-command mode; sequence cases should use
+        ``execute_sequence()`` instead.
+        """
+        return {
+            "name": self.name,
+            "command": self.command,
+            "args": self.args,
+            "expected": self.expected,
+            "description": self.description or None,
+            "timeout": self.timeout,
+            "resources": self.resources,
+            "retry_count": self.retry_count,
+        }

symtest/core/types.py

@@ -0,0 +1,92 @@
+from typing import Any, Dict, List, Optional, TypedDict
+
+
+class ExpectedResult(TypedDict, total=False):
+    """Expectation configuration for a single test case."""
+
+    return_code: Optional[int]
+    output_contains: List[str]
+    output_matches: Optional[str]
+    compare_files: List[Dict[str, Any]]
+    """List of file comparison specs.
+    Each spec is a dict with:
+        actual   (str): path to the file produced by the test command
+        baseline (str): path to the golden/reference file
+        type     (str, optional): comparator type (e.g. 'h5','json','csv','xml','text','binary')
+                                  auto-detected from extension if omitted
+        Additional kwargs are forwarded to the comparator (e.g. rtol, atol, encoding, ...)
+    """
+
+
+class ResourceRequirements(TypedDict, total=False):
+    """Optional resource hints for scheduling."""
+
+    estimated_time: float  # seconds, used for ordering (LPT)
+    min_memory_mb: float  # soft hint to avoid OOM
+    priority: int  # higher value => higher priority
+    cpu_cores: int  # number of CPU cores required by this task
+
+
+class TestCaseData(TypedDict, total=False):
+    """Input data shape for a test case after解析/路径处理."""
+
+    name: str
+    command: str
+    args: List[str]
+    expected: ExpectedResult
+    description: Optional[str]
+    timeout: Optional[float]
+    resources: Optional[ResourceRequirements]
+    retry_count: int
+    expected_failure: bool
+    xfail_reason: Optional[str]
+    xfail_quiet: bool
+
+
+class SetupConfig(TypedDict):
+    """Setup configuration (currently environment variables only)."""
+
+    environment_variables: Dict[str, str]
+
+
+class TestSuiteConfig(TypedDict):
+    """Top-level configuration for a suite loaded from JSON/YAML."""
+
+    setup: Optional[SetupConfig]
+    test_cases: List[TestCaseData]
+
+
+class TestResultData(TypedDict):
+    """Normalized result produced by executing a single test case."""
+
+    name: str
+    status: str  # 'passed', 'failed', 'timeout'
+    message: str
+    command: str
+    output: str
+    return_code: Optional[int]
+    duration: float
+    # ── New fields (optional, backward-compatible) ──
+    expected: Optional[Dict[str, Any]]
+    description: Optional[str]
+    tags: List[str]
+    failure_kind: Optional[str]  # 'return_code' | 'output_contains' | 'output_matches' | 'file_compare' | 'timeout' | 'execution_error'
+    attempts: int
+    flaky: bool
+    attempt_history: List[Dict[str, Any]]
+    step_results: List[Dict[str, Any]]
+    compare_failures: List[Dict[str, Any]]
+    baseline_updated: List[str]
+    failed_step: Optional[int]
+    # ── Structured output channels (split from combined ``output``) ──
+    stdout: str
+    stderr: str
+    # ── Per-assertion pass/fail detail (populated on both pass and failure) ──
+    assertion_results: List[Dict[str, Any]]
+    xfail_reason: Optional[str]
+    # ── Structured remediation suggestion (populated on failure) ──
+    # Dict with: action ('update_baseline' | 'update_expected' | 'increase_timeout'
+    # | 'investigate'), command (concrete symtest command, filled by runners),
+    # reason (human/AI-readable explanation).
+    next_action_hint: Optional[Dict[str, Any]]
+

symtest/file_comparator/__init__.py

@@ -0,0 +1,10 @@
+"""
+File comparison module for symtest.
+This module provides functionality for comparing different types of files.
+"""
+
+from .factory import ComparatorFactory
+from .result import ComparisonResult
+from .base_comparator import BaseComparator
+
+__all__ = ['ComparatorFactory', 'ComparisonResult', 'BaseComparator']

symtest/file_comparator/base_comparator.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+@file base_comparator.py
+@brief Base abstract class for file comparison operations
+@author Xiaotong Wang
+@date 2025
+"""
+
+from abc import ABC, abstractmethod
+import logging
+from pathlib import Path
+from .result import ComparisonResult, Difference
+
+class BaseComparator(ABC):
+    """
+    @brief Base abstract class for all file comparators
+    @details This class defines the interface and common functionality for all file comparators.
+             It provides basic file comparison operations and logging capabilities.
+    """
+    
+    def __init__(self, encoding="utf-8", chunk_size=8192, verbose=False, **kwargs):
+        """
+        @brief Initialize the base comparator
+        @param encoding str: File encoding to use (default: "utf-8")
+        @param chunk_size int: Size of chunks for reading large files (default: 8192)
+        @param verbose bool: Enable verbose logging (default: False)
+        """
+        self.encoding = encoding
+        self.chunk_size = chunk_size
+        self.logger = logging.getLogger(f"file_comparator.{self.__class__.__name__}")
+        if verbose:
+            self.logger.setLevel(logging.DEBUG)
+    
+    @abstractmethod
+    def read_content(self, file_path, start_line=0, end_line=None, start_column=0, end_column=None):
+        """
+        @brief Read file content with specified range
+        @param file_path Path: Path to the file to read
+        @param start_line int: Starting line number (0-based)
+        @param end_line int: Ending line number (0-based, None for end of file)
+        @param start_column int: Starting column number (0-based)
+        @param end_column int: Ending column number (0-based, None for end of line)
+        @return object: File content in a format suitable for comparison
+        """
+        pass
+
+    @abstractmethod
+    def compare_content(self, content1, content2):
+        """
+        @brief Compare two content objects and return comparison details
+        @param content1 object: First content object to compare
+        @param content2 object: Second content object to compare
+        @return tuple: (bool, list, bool) - (identical, differences, truncated)
+        """
+        pass
+
+    def compare_files(self, file1, file2, start_line=0, end_line=None, start_column=0, end_column=None):
+        """
+        @brief Compare two files with the specified parameters
+        @param file1 Path: Path to the first file
+        @param file2 Path: Path to the second file
+        @param start_line int: Starting line number (0-based)
+        @param end_line int: Ending line number (0-based, None for end of file)
+        @param start_column int: Starting column number (0-based)
+        @param end_column int: Ending column number (0-based, None for end of line)
+        @return ComparisonResult: Result object containing comparison details
+        """
+        result = ComparisonResult(
+            file1=str(file1),
+            file2=str(file2),
+            start_line=start_line,
+            end_line=end_line,
+            start_column=start_column,
+            end_column=end_column
+        )
+        
+        try:
+            self.logger.info(f"Comparing files: {file1} and {file2}")
+            
+            # Record file metadata
+            file1_path = Path(file1)
+            file2_path = Path(file2)
+            result.file1_size = file1_path.stat().st_size
+            result.file2_size = file2_path.stat().st_size
+            
+            # Read content with specified ranges
+            self.logger.debug(f"Reading content from files")
+            content1 = self.read_content(file1, start_line, end_line, start_column, end_column)
+            content2 = self.read_content(file2, start_line, end_line, start_column, end_column)
+            
+            # Compare content
+            self.logger.debug(f"Comparing content")
+            identical, differences, truncated = self.compare_content(content1, content2)
+            
+            # Update result
+            result.identical = identical
+            result.differences = differences
+            result.truncated = truncated
+            result.error_stats = getattr(self, "_error_stats", None)
+
+            return result
+            
+        except Exception as e:
+            self.logger.error(f"Error during comparison: {str(e)}")
+            result.error = str(e)
+            result.identical = False
+            return result