cli-test-framework 0.3.7__tar.gz → 0.4.1__tar.gz

{cli_test_framework-0.3.7/src/cli_test_framework.egg-info → cli_test_framework-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cli-test-framework
-Version: 0.3.7
+Version: 0.4.1
 Summary: A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.
 Home-page: https://github.com/ozil111/cli-test-framework
 Author: Xiaotong Wang
@@ -53,6 +53,7 @@ This is a lightweight and extensible automated testing framework that supports d
 - **📊 Comprehensive Reports**: Detailed pass rate statistics and failure diagnostics
 - **🔧 Thread-Safe Design**: Robust concurrent execution with proper synchronization
 - **📝 Advanced File Comparison**: Support for comparing various file types (text, binary, JSON, HDF5) with detailed diff output
+- **🎛️ Resource-Aware Scheduling**: Per-test timeout and resource hints (estimated time / memory / priority) with LPT-based ordering in parallel runs to improve throughput and avoid long-tail blocking
 
 ## 3. Quick Start
 
@@ -144,6 +145,20 @@ compare-files binary1.bin binary2.bin --similarity
                 "output_contains": ["Environment: development"]
             }
         },
+        {
+            "name": "Full_Car_Crash_Simulation",
+            "command": "radioss_solver",
+            "args": ["-i", "input.0000.rad"],
+            "timeout": 36000,
+            "resources": {
+                "estimated_time": 18000,
+                "min_memory_mb": 16000,
+                "priority": 10
+            },
+            "expected": {
+                "return_code": 0
+            }
+        },
         {
             "name": "File Comparison Test", 
             "command": "compare-files",

{cli_test_framework-0.3.7 → cli_test_framework-0.4.1}/README.md

@@ -18,6 +18,7 @@ This is a lightweight and extensible automated testing framework that supports d
 - **📊 Comprehensive Reports**: Detailed pass rate statistics and failure diagnostics
 - **🔧 Thread-Safe Design**: Robust concurrent execution with proper synchronization
 - **📝 Advanced File Comparison**: Support for comparing various file types (text, binary, JSON, HDF5) with detailed diff output
+- **🎛️ Resource-Aware Scheduling**: Per-test timeout and resource hints (estimated time / memory / priority) with LPT-based ordering in parallel runs to improve throughput and avoid long-tail blocking
 
 ## 3. Quick Start
 
@@ -109,6 +110,20 @@ compare-files binary1.bin binary2.bin --similarity
                 "output_contains": ["Environment: development"]
             }
         },
+        {
+            "name": "Full_Car_Crash_Simulation",
+            "command": "radioss_solver",
+            "args": ["-i", "input.0000.rad"],
+            "timeout": 36000,
+            "resources": {
+                "estimated_time": 18000,
+                "min_memory_mb": 16000,
+                "priority": 10
+            },
+            "expected": {
+                "return_code": 0
+            }
+        },
         {
             "name": "File Comparison Test", 
             "command": "compare-files",

{cli_test_framework-0.3.7 → cli_test_framework-0.4.1}/setup.py

@@ -8,7 +8,7 @@ with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name="cli-test-framework",
-    version="0.3.7",
+    version="0.4.1",
     author="Xiaotong Wang",
     author_email="xiaotongwang98@gmail.com",
     description="A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.",

cli_test_framework-0.4.1/src/cli_test_framework/core/execution.py

@@ -0,0 +1,74 @@
+import subprocess
+import time
+from typing import Optional
+
+from .assertions import Assertions
+from .types import ExpectedResult, TestCaseData, TestResultData
+
+
+def validate_result(expected: ExpectedResult, actual: TestResultData) -> None:
+    """
+    Pure validation logic. Raises AssertionError on mismatch.
+    """
+    assertions = Assertions()
+
+    if "return_code" in expected:
+        assertions.return_code_equals(actual["return_code"], expected["return_code"])
+
+    if "output_contains" in expected:
+        for text in expected["output_contains"]:
+            assertions.contains(actual["output"], text)
+
+    if "output_matches" in expected and expected["output_matches"]:
+        assertions.matches(actual["output"], expected["output_matches"])
+
+
+def execute_single_test_case(case: TestCaseData, workspace: Optional[str] = None) -> TestResultData:
+    """
+    Stateless execution of a single test case.
+    """
+    start_time = time.time()
+    full_command = f"{case['command']} {' '.join(case['args'])}".strip()
+    timeout_limit = case.get("timeout", 3600)
+
+    result: TestResultData = {
+        "name": case["name"],
+        "status": "failed",
+        "message": "",
+        "command": full_command,
+        "output": "",
+        "return_code": None,
+        "duration": 0.0,
+    }
+
+    try:
+        process = subprocess.run(
+            full_command,
+            cwd=workspace if workspace else None,
+            capture_output=True,
+            text=True,
+            check=False,
+            shell=True,
+            timeout=timeout_limit if timeout_limit is not None else None,
+        )
+
+        output = process.stdout + process.stderr
+        result["output"] = output
+        result["return_code"] = process.returncode
+
+        validate_result(case["expected"], result)
+        result["status"] = "passed"
+    except subprocess.TimeoutExpired as exc:
+        result["status"] = "timeout"
+        result["message"] = f"Timeout reached! Killed after {timeout_limit} seconds."
+        result["output"] = (exc.stdout or "") + (exc.stderr or "")
+        result["return_code"] = None
+    except AssertionError as exc:
+        result["message"] = str(exc)
+    except Exception as exc:
+        result["message"] = f"Execution error: {str(exc)}"
+    finally:
+        result["duration"] = time.time() - start_time
+
+    return result
+

cli_test_framework-0.4.1/src/cli_test_framework/core/process_worker.py

@@ -0,0 +1,43 @@
+"""
+进程工作器模块
+用于多进程并行测试执行，避免序列化问题
+"""
+
+from typing import Dict, Any
+from .execution import execute_single_test_case
+from .types import TestCaseData
+
+def run_test_in_process(test_index: int, case_data: Dict[str, Any], workspace: str = None) -> Dict[str, Any]:
+    """
+    在独立进程中运行单个测试用例
+    
+    Args:
+        test_index: 测试索引
+        case_data: 测试用例数据字典
+        workspace: 工作目录
+    
+    Returns:
+        测试结果字典
+    """
+    case: TestCaseData = {
+        "name": case_data["name"],
+        "command": case_data["command"],
+        "args": case_data["args"],
+        "expected": case_data["expected"],
+        "description": case_data.get("description"),
+    }
+
+    command_preview = f"{case['command']} {' '.join(case['args'])}".strip()
+    print(f"  [Process Worker {test_index}] Executing command: {command_preview}")
+
+    result = execute_single_test_case(case, workspace)
+
+    if result["output"].strip():
+        print(f"  [Process Worker {test_index}] Command output for {case['name']}:")
+        for line in result["output"].splitlines():
+            print(f"    {line}")
+
+    if result["status"] != "passed" and result.get("message"):
+        print(f"  [Process Worker {test_index}] Error for {case['name']}: {result['message']}")
+
+    return result 

{cli_test_framework-0.3.7 → cli_test_framework-0.4.1}/src/cli_test_framework/core/test_case.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Optional
 
 @dataclass
 class TestCase:
@@ -8,6 +8,8 @@ class TestCase:
     args: List[str]
     expected: Dict[str, Any]
     description: str = ""
+    timeout: Optional[float] = None
+    resources: Optional[Dict[str, Any]] = None
     
     def to_dict(self) -> Dict[str, Any]:
         """Convert test case to dictionary format"""
@@ -17,5 +19,7 @@ class TestCase:
             "name": self.name,
             "command": self.command,
             "args": self.args,
-            "expected": self.expected
+            "expected": self.expected,
+            "timeout": self.timeout,
+            "resources": self.resources,
         }

cli_test_framework-0.4.1/src/cli_test_framework/core/types.py

@@ -0,0 +1,55 @@
+from typing import Any, Dict, List, Optional, TypedDict
+
+
+class ExpectedResult(TypedDict, total=False):
+    """Expectation configuration for a single test case."""
+
+    return_code: Optional[int]
+    output_contains: List[str]
+    output_matches: Optional[str]
+
+
+class ResourceRequirements(TypedDict, total=False):
+    """Optional resource hints for scheduling."""
+
+    estimated_time: float  # seconds, used for ordering (LPT)
+    min_memory_mb: float  # soft hint to avoid OOM
+    priority: int  # higher value => higher priority
+
+
+class TestCaseData(TypedDict):
+    """Input data shape for a test case after解析/路径处理."""
+
+    name: str
+    command: str
+    args: List[str]
+    expected: ExpectedResult
+    description: Optional[str]
+    timeout: Optional[float]
+    resources: Optional[ResourceRequirements]
+
+
+class SetupConfig(TypedDict):
+    """Setup configuration (currently environment variables only)."""
+
+    environment_variables: Dict[str, str]
+
+
+class TestSuiteConfig(TypedDict):
+    """Top-level configuration for a suite loaded from JSON/YAML."""
+
+    setup: Optional[SetupConfig]
+    test_cases: List[TestCaseData]
+
+
+class TestResultData(TypedDict):
+    """Normalized result produced by executing a single test case."""
+
+    name: str
+    status: str  # 'passed', 'failed'
+    message: str
+    command: str
+    output: str
+    return_code: Optional[int]
+    duration: float
+

{cli_test_framework-0.3.7 → cli_test_framework-0.4.1}/src/cli_test_framework/file_comparator/json_comparator.py

@@ -53,8 +53,8 @@ class JsonComparator(TextComparator):
         json_text = ''.join(text_content)
         try:
             json_data = json.loads(json_text)
-            if self.key_field:
-                # Only keep the specified key field(s)
+            if self.key_field and isinstance(json_data, dict):
+                # Only keep the specified key field(s) when top-level is a mapping
                 key_fields = self.key_field if isinstance(self.key_field, list) else [self.key_field]
                 filtered_data = {key: json_data[key] for key in key_fields if key in json_data}
                 return filtered_data
@@ -79,8 +79,9 @@ class JsonComparator(TextComparator):
             self._compare_json_key_based(content1, content2, "", differences)
         else:
             self._compare_json_exact(content1, content2, "", differences)
-            
-        return False, differences
+        
+        identical = len(differences) == 0
+        return identical, differences
 
     def _compare_json_exact(self, obj1, obj2, path, differences, max_diffs=10):
         """

cli_test_framework-0.4.1/src/cli_test_framework/runners/json_runner.py

@@ -0,0 +1,65 @@
+from typing import Optional, Dict, Any
+from ..core.base_runner import BaseRunner
+from ..core.test_case import TestCase
+from ..core.execution import execute_single_test_case
+from ..core.types import TestCaseData
+from ..utils.path_resolver import PathResolver, parse_command_string, resolve_paths
+import json
+import sys
+
+class JSONRunner(BaseRunner):
+    def __init__(self, config_file="test_cases.json", workspace: Optional[str] = None):
+        super().__init__(config_file, workspace)
+        # Backward-compatible attribute for tests that patch path_resolver
+        self.path_resolver = PathResolver(self.workspace)
+
+    def load_test_cases(self) -> None:
+        """Load test cases from a JSON file."""
+        try:
+            with open(self.config_path, 'r', encoding='utf-8') as f:
+                config = json.load(f)
+
+            # 加载setup配置
+            self.load_setup_from_config(config)
+
+            required_fields = ["name", "command", "args", "expected"]
+            for case in config["test_cases"]:
+                if not all(field in case for field in required_fields):
+                    raise ValueError(f"Test case {case.get('name', 'unnamed')} is missing required fields")
+
+                # 使用智能命令解析，正确处理包含空格的路径
+                # Use resolver attribute (keeps backward compatibility with tests monkeypatching it)
+                case["command"] = self.path_resolver.parse_command_string(case["command"])
+                case["args"] = self.path_resolver.resolve_paths(case["args"])
+                self.test_cases.append(TestCase(**case))
+
+            print(f"Successfully loaded {len(self.test_cases)} test cases")
+            # print(self.test_cases)
+        except Exception as e:
+            sys.exit(f"Failed to load configuration file: {str(e)}")
+
+    def run_single_test(self, case: TestCase) -> Dict[str, str]:
+        case_data: TestCaseData = {
+            "name": case.name,
+            "command": case.command,
+            "args": case.args,
+            "expected": case.expected,
+            "description": case.description or None,
+            "timeout": case.timeout,
+            "resources": case.resources,
+        }
+
+        command_preview = f"{case_data['command']} {' '.join(case_data['args'])}".strip()
+        print(f"  Executing command: {command_preview}")
+
+        result = execute_single_test_case(case_data, str(self.workspace) if self.workspace else None)
+
+        if result["output"].strip():
+            print("  Command output:")
+            for line in result["output"].splitlines():
+                print(f"    {line}")
+
+        if result["status"] != "passed" and result.get("message"):
+            print(f"  Error: {result['message']}")
+
+        return result

cli_test_framework-0.4.1/src/cli_test_framework/runners/parallel_json_runner.py

@@ -0,0 +1,92 @@
+from typing import Optional, Dict, Any
+from ..core.parallel_runner import ParallelRunner
+from ..core.test_case import TestCase
+from ..core.execution import execute_single_test_case
+from ..core.types import TestCaseData
+from ..utils.path_resolver import PathResolver, parse_command_string, resolve_paths
+import json
+import sys
+import threading
+
+class ParallelJSONRunner(ParallelRunner):
+    """并行JSON测试运行器"""
+    
+    def __init__(self, config_file="test_cases.json", workspace: Optional[str] = None,
+                 max_workers: Optional[int] = None, execution_mode: str = "thread"):
+        """
+        初始化并行JSON运行器
+        
+        Args:
+            config_file: JSON配置文件路径
+            workspace: 工作目录
+            max_workers: 最大并发数
+            execution_mode: 执行模式，'thread' 或 'process'
+        """
+        super().__init__(config_file, workspace, max_workers, execution_mode)
+        # Backward-compatible attribute for potential external patches/tests
+        self.path_resolver = PathResolver(self.workspace)
+        self._print_lock = threading.Lock()  # 用于控制输出顺序
+
+    def load_test_cases(self) -> None:
+        """从JSON文件加载测试用例"""
+        try:
+            with open(self.config_path, 'r', encoding='utf-8') as f:
+                config = json.load(f)
+
+            # 加载setup配置
+            self.load_setup_from_config(config)
+
+            required_fields = ["name", "command", "args", "expected"]
+            for case in config["test_cases"]:
+                if not all(field in case for field in required_fields):
+                    raise ValueError(f"Test case {case.get('name', 'unnamed')} is missing required fields")
+
+                # Use resolver attribute (keeps backward compatibility with tests monkeypatching it)
+                case["command"] = self.path_resolver.parse_command_string(case["command"])
+                case["args"] = self.path_resolver.resolve_paths(case["args"])
+                self.test_cases.append(TestCase(**case))
+
+            print(f"Successfully loaded {len(self.test_cases)} test cases")
+
+            # Heuristic scheduling: longest estimated time first to improve parallel utilization.
+            if self.test_cases:
+                print("Optimizing execution order based on estimated duration...")
+                self.test_cases.sort(
+                    key=lambda c: (c.resources or {}).get("estimated_time", 0),
+                    reverse=True,
+                )
+                top_case = self.test_cases[0]
+                top_est = (top_case.resources or {}).get("estimated_time", 0)
+                print(f"Heaviest task: {top_case.name} (Est: {top_est}s)")
+        except Exception as e:
+            sys.exit(f"Failed to load configuration file: {str(e)}")
+
+    def run_single_test(self, case: TestCase) -> Dict[str, Any]:
+        """运行单个测试用例（线程安全版本）"""
+        case_data: TestCaseData = {
+            "name": case.name,
+            "command": case.command,
+            "args": case.args,
+            "expected": case.expected,
+            "description": case.description or None,
+            "timeout": case.timeout,
+            "resources": case.resources,
+        }
+
+        command_preview = f"{case_data['command']} {' '.join(case_data['args'])}".strip()
+        with self._print_lock:
+            print(f"  [Worker] Executing command: {command_preview}")
+
+        result = execute_single_test_case(case_data, str(self.workspace) if self.workspace else None)
+
+        if result["output"].strip():
+            with self._print_lock:
+                print(f"  [Worker] Command output for {case.name}:")
+                for line in result["output"].splitlines():
+                    print(f"    {line}")
+
+        if result["status"] != "passed" and result.get("message"):
+            with self._print_lock:
+                print(f"  [Worker] Error for {case.name}: {result['message']}")
+
+        return result 

cli_test_framework-0.4.1/src/cli_test_framework/runners/yaml_runner.py

@@ -0,0 +1,64 @@
+from typing import Optional, Dict, Any
+from ..core.base_runner import BaseRunner
+from ..core.test_case import TestCase
+from ..core.execution import execute_single_test_case
+from ..core.types import TestCaseData
+from ..utils.path_resolver import PathResolver, parse_command_string, resolve_paths
+import sys
+
+class YAMLRunner(BaseRunner):
+    def __init__(self, config_file="test_cases.yaml", workspace: Optional[str] = None):
+        super().__init__(config_file, workspace)
+        # Backward-compatible attribute for tests that patch path_resolver
+        self.path_resolver = PathResolver(self.workspace)
+
+    def load_test_cases(self):
+        """Load test cases from a YAML file."""
+        try:
+            import yaml
+            with open(self.config_path, 'r', encoding='utf-8') as f:
+                config = yaml.safe_load(f)
+                
+            # 加载setup配置
+            self.load_setup_from_config(config)
+                
+            required_fields = ["name", "command", "args", "expected"]
+            for case in config["test_cases"]:
+                if not all(field in case for field in required_fields):
+                    raise ValueError(f"Test case {case.get('name', 'unnamed')} is missing required fields")
+                
+                # Use resolver attribute (keeps backward compatibility with tests monkeypatching it)
+                case["command"] = self.path_resolver.parse_command_string(case["command"])
+                case["args"] = self.path_resolver.resolve_paths(case["args"])
+                self.test_cases.append(TestCase(**case))
+                
+            print(f"Successfully loaded {len(self.test_cases)} test cases")
+        except Exception as e:
+            sys.exit(f"Failed to load configuration file: {str(e)}")
+
+    def run_single_test(self, case: TestCase) -> Dict[str, Any]:
+        """Run a single test case and return the result"""
+        case_data: TestCaseData = {
+            "name": case.name,
+            "command": case.command,
+            "args": case.args,
+            "expected": case.expected,
+            "description": case.description or None,
+            "timeout": case.timeout,
+            "resources": case.resources,
+        }
+
+        command_preview = f"{case_data['command']} {' '.join(case_data['args'])}".strip()
+        print(f"  Executing command: {command_preview}")
+
+        result = execute_single_test_case(case_data, str(self.workspace) if self.workspace else None)
+
+        if result["output"].strip():
+            print("  Command output:")
+            for line in result["output"].splitlines():
+                print(f"    {line}")
+
+        if result["status"] != "passed" and result.get("message"):
+            print(f"  Error: {result['message']}")
+
+        return result

cli_test_framework-0.4.1/src/cli_test_framework/utils/__init__.py

@@ -0,0 +1,19 @@
+# File: /python-test-framework/python-test-framework/src/utils/__init__.py
+
+"""
+Utility functions for the CLI Testing Framework
+"""
+
+from .path_resolver import (
+    PathResolver,
+    parse_command_string,
+    resolve_paths,
+    resolve_command,
+)
+
+__all__ = [
+    'PathResolver',
+    'parse_command_string',
+    'resolve_paths',
+    'resolve_command',
+]