cli-test-framework 1.0.1__tar.gz → 1.0.2__tar.gz

{cli_test_framework-1.0.1/src/cli_test_framework.egg-info → cli_test_framework-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cli-test-framework
-Version: 1.0.1
+Version: 1.0.2
 Summary: A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.
 Home-page: https://github.com/ozil111/cli-test-framework
 Author: Xiaotong Wang

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/docs/user_manual.md

@@ -49,6 +49,7 @@ HDF5 文件比较依赖 `h5py`（已随框架安装）。如需在无 HDF5 环
             "command": "echo",
             "args": ["Hello"],
             "timeout": 60,
+            "retry_count": 0,
             "resources": {
                 "cpu_cores": 2,
                 "estimated_time": 300,
@@ -80,6 +81,7 @@ test_cases:
     command: echo
     args: ["Hello"]
     timeout: 60
+    retry_count: 0
     resources:
       cpu_cores: 2
       estimated_time: 300
@@ -105,6 +107,7 @@ test_cases:
 | `args` | 否 | 命令参数列表 |
 | `description` | 否 | 测试用例描述 |
 | `timeout` | 否 | 超时秒数，默认 3600，设 `null` 无限制 |
+| `retry_count` | 否 | 失败自动重试次数，默认 0（不重试）。单命令模式作用于整个 case，步骤模式可作用于每个 step |
 | `tags` | 否 | 标签列表，用于批量过滤（如 `["smoke", "fast"]`） |
 | `resources` | 否 | 资源配置，见[资源感知调度](#资源感知调度) |
 | `expected.return_code` | 否 | 期望返回码 |
@@ -801,7 +804,8 @@ runner.run_tests_sequential()
                 {
                     "command": "echo",
                     "args": ["step2"],
-                    "expected": { "return_code": 0 }
+                    "expected": { "return_code": 0 },
+                    "retry_count": 2
                 }
             ]
         }
@@ -825,7 +829,7 @@ test_cases:
           return_code: 0
 ```
 
-每个 step 支持 `command`、`args`、`expected`、`timeout` 字段。失败时结果会标注失败步骤编号，如 "Failed at step 2/3"。
+每个 step 支持 `command`、`args`、`expected`、`timeout`、`retry_count` 字段。失败时结果会标注失败步骤编号，如 "Failed at step 2/3"。
 
 ## 资源感知调度
 

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/setup.py

@@ -8,7 +8,7 @@ with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name="cli-test-framework",
-    version="1.0.1",
+    version="1.0.2",
     author="Xiaotong Wang",
     author_email="xiaotongwang98@gmail.com",
     description="A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.",

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/core/base_runner.py

@@ -129,11 +129,14 @@ class BaseRunner(ABC):
             self.setup_manager.teardown_all()
 
     def _update_history(self) -> None:
-        """Update .symtest history with current run results and check for regressions."""
+        """Update .symtest history with successful run results and check for regressions."""
         if not self.history_dir:
             return
         history = load_history(self.history_dir)
         for result in self.results["details"]:
+            # Only record successful cases in history; skip failed ones
+            if result["status"] != "passed":
+                continue
             duration = result.get("duration", 0)
             # Check regression BEFORE updating (compare against old avg)
             warning = check_regression(history, result["name"], duration, self.regression_threshold)

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/core/config_loader.py

@@ -115,6 +115,7 @@ def parse_test_cases(
                         args=resolved_args,
                         expected=step["expected"],
                         timeout=step.get("timeout"),
+                        retry_count=step.get("retry_count", 0),
                     ))
                 else:
                     steps.append(TestCaseStep(
@@ -122,6 +123,7 @@ def parse_test_cases(
                         args=step.get("args", []),
                         expected=step.get("expected", {}),
                         timeout=step.get("timeout"),
+                        retry_count=step.get("retry_count", 0),
                     ))
             cases.append(TestCase(
                 name=case.get("name", ""),
@@ -151,6 +153,7 @@ def parse_test_cases(
                     timeout=case.get("timeout"),
                     resources=case.get("resources"),
                     tags=case.get("tags", []),
+                    retry_count=case.get("retry_count", 0),
                 ))
             else:
                 cases.append(TestCase(
@@ -162,6 +165,7 @@ def parse_test_cases(
                     timeout=case.get("timeout"),
                     resources=case.get("resources"),
                     tags=case.get("tags", []),
+                    retry_count=case.get("retry_count", 0),
                 ))
 
     return cases
@@ -236,6 +240,7 @@ def execute_sequence(
             "description": None,
             "timeout": _step_attr(step, "timeout"),
             "resources": None,
+            "retry_count": _step_attr(step, "retry_count", 0),
         }
 
         command_preview = (

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/core/execution.py

@@ -3,11 +3,14 @@ import signal
 import time
 import os
 import shlex
+import logging
 from typing import Any, List, Optional, Dict
 
 from .assertions import Assertions
 from .types import ExpectedResult, TestCaseData, TestResultData
 
+logger = logging.getLogger("cli_test_framework.core.execution")
+
 # Commands that are shell builtins (not real executables).
 # With shell=False, these must be wrapped via the platform shell.
 if os.name == 'nt':
@@ -80,15 +83,12 @@ def _dispatch_file_compare(
     )
 
 
-def execute_single_test_case(case: TestCaseData, workspace: Optional[str] = None, env: Optional[Dict[str, str]] = None) -> TestResultData:
-    """
-    Stateless execution of a single test case.
-    
-    Args:
-        case: Test case data
-        workspace: Working directory for test execution
-        env: Optional environment variables to inject/override (merged with os.environ)
-    """
+def _execute_command_once(
+    case: TestCaseData,
+    workspace: Optional[str] = None,
+    env: Optional[Dict[str, str]] = None,
+) -> TestResultData:
+    """Execute a single command once (no retry logic)."""
     start_time = time.time()
     cmd_list = _normalize_cmd_list(case["command"], [str(arg) for arg in case["args"]])
     timeout_limit = case.get("timeout", 3600)
@@ -157,3 +157,43 @@ def execute_single_test_case(case: TestCaseData, workspace: Optional[str] = None
 
     return result
 
+
+def execute_single_test_case(
+    case: TestCaseData,
+    workspace: Optional[str] = None,
+    env: Optional[Dict[str, str]] = None,
+) -> TestResultData:
+    """
+    Stateless execution of a single test case with optional retry.
+
+    Args:
+        case: Test case data (may include ``retry_count`` for automatic retries).
+        workspace: Working directory for test execution.
+        env: Optional environment variables to inject/override (merged with os.environ).
+
+    ``retry_count`` (defaults to 0) controls how many additional times the
+    command is re-run after the first failure.  ``retry_count=0`` means no
+    retry (behaviour identical to previous versions).
+    """
+    retry_count: int = case.get("retry_count", 0)
+    max_attempts = retry_count + 1
+    total_duration = 0.0
+    last_result: Optional[TestResultData] = None
+
+    for attempt in range(1, max_attempts + 1):
+        result = _execute_command_once(case, workspace, env)
+        total_duration += result["duration"]
+        last_result = result
+
+        if result["status"] == "passed":
+            break
+
+        if attempt < max_attempts:
+            logger.info(
+                "Retrying '%s' (attempt %d/%d)...",
+                case["name"], attempt + 1, max_attempts,
+            )
+
+    last_result["duration"] = total_duration
+    return last_result
+

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/core/parallel_runner.py

@@ -138,12 +138,14 @@ class ParallelRunner(BaseRunner):
                                 "expected": case.expected,
                                 "timeout": case.timeout,
                                 "resources": case.resources,
+                                "retry_count": case.retry_count,
                                 "steps": [
                                     {
                                         "command": s.command,
                                         "args": s.args,
                                         "expected": s.expected,
                                         "timeout": s.timeout,
+                                        "retry_count": s.retry_count,
                                     }
                                     for s in case.steps
                                 ] if case.steps else None,

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/core/process_worker.py

@@ -46,6 +46,7 @@ def run_test_in_process(test_index: int, case_data: Dict[str, Any], workspace: s
         "description": case_data.get("description"),
         "timeout": case_data.get("timeout"),
         "resources": case_data.get("resources"),
+        "retry_count": case_data.get("retry_count", 0),
     }
 
     command_preview = f"{case['command']} {' '.join(case['args'])}".strip()

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/core/test_case.py

@@ -9,6 +9,7 @@ class TestCaseStep:
     args: List[str]
     expected: Dict[str, Any]
     timeout: Optional[float] = None
+    retry_count: int = 0
 
 @dataclass
 class TestCase:
@@ -22,6 +23,7 @@ class TestCase:
     resources: Optional[Dict[str, Any]] = None
     steps: Optional[List[TestCaseStep]] = None
     tags: List[str] = field(default_factory=list)
+    retry_count: int = 0
     
     def to_dict(self) -> Dict[str, Any]:
         """Convert test case to dictionary format"""
@@ -33,6 +35,7 @@ class TestCase:
             "timeout": self.timeout,
             "resources": self.resources,
             "tags": self.tags,
+            "retry_count": self.retry_count,
         }
         if self.steps is not None:
             result["steps"] = [
@@ -41,6 +44,7 @@ class TestCase:
                     "args": s.args,
                     "expected": s.expected,
                     "timeout": s.timeout,
+                    "retry_count": s.retry_count,
                 }
                 for s in self.steps
             ]
@@ -60,4 +64,5 @@ class TestCase:
             "description": self.description or None,
             "timeout": self.timeout,
             "resources": self.resources,
+            "retry_count": self.retry_count,
         }

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/core/types.py

@@ -37,6 +37,7 @@ class TestCaseData(TypedDict):
     description: Optional[str]
     timeout: Optional[float]
     resources: Optional[ResourceRequirements]
+    retry_count: int
 
 
 class SetupConfig(TypedDict):

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/tui/screens/case_editor.py

@@ -90,6 +90,8 @@ class CaseEditorScreen(Screen):
                 TextArea("", id="case-description"),
                 Label("Timeout (seconds):", id="timeout-label"),
                 Input(placeholder="", id="case-timeout"),
+                Label("Retry count:", id="retry-label"),
+                Input(placeholder="0", id="case-retry-count"),
                 ExpectedEditor(id="expected-editor"),
                 StepsEditor(id="steps-editor"),
                 id="editor-fields",
@@ -127,6 +129,7 @@ class CaseEditorScreen(Screen):
         self.query_one("#case-tags", Input).value = ",".join(tc.tags) if tc.tags else ""
         self.query_one("#case-description", TextArea).text = tc.description or ""
         self.query_one("#case-timeout", Input).value = str(tc.timeout) if tc.timeout else ""
+        self.query_one("#case-retry-count", Input).value = str(tc.retry_count) if tc.retry_count else ""
 
         self.query_one("#expected-editor", ExpectedEditor).load(tc.expected or {})
 
@@ -147,6 +150,9 @@ class CaseEditorScreen(Screen):
         timeout_text = self.query_one("#case-timeout", Input).value.strip()
         timeout = float(timeout_text) if timeout_text else None
 
+        retry_text = self.query_one("#case-retry-count", Input).value.strip()
+        retry_count = int(retry_text) if retry_text else 0
+
         description = self.query_one("#case-description", TextArea).text.strip()
 
         if self._is_sequence:
@@ -157,6 +163,7 @@ class CaseEditorScreen(Screen):
                 description=description or "",
                 resources=self._case.resources if self._case else None,
                 tags=tags,
+                retry_count=retry_count,
             )
         else:
             command = self.query_one("#case-command", Input).value.strip()
@@ -173,6 +180,7 @@ class CaseEditorScreen(Screen):
                 timeout=timeout,
                 resources=self._case.resources if self._case else None,
                 tags=tags,
+                retry_count=retry_count,
             )
 
     def _update_mode_visibility(self) -> None:

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework/tui/widgets/steps_editor.py

@@ -74,6 +74,8 @@ class StepsEditor(Vertical):
         yield TextArea('{"return_code": 0}', id="step-expected")
         yield Static("Timeout (seconds):", classes="label")
         yield Input(placeholder="", id="step-timeout")
+        yield Static("Retry count:", classes="label")
+        yield Input(placeholder="0", id="step-retry-count")
         yield Horizontal(
             Button("Save Step", id="step-save", variant="primary"),
             Button("Cancel", id="step-cancel"),
@@ -114,6 +116,7 @@ class StepsEditor(Vertical):
             if len(exp_preview) > 40:
                 exp_preview = exp_preview[:37] + "..."
             timeout_str = f", timeout={step.timeout}s" if step.timeout else ""
+            retry_str = f", retry={step.retry_count}" if step.retry_count else ""
 
             row = Horizontal(
                 Static(f"[Step {i+1}/{len(self._steps)}]"),
@@ -136,6 +139,7 @@ class StepsEditor(Vertical):
         self.query_one("#step-args", Input).value = ""
         self.query_one("#step-expected", TextArea).text = '{"return_code": 0}'
         self.query_one("#step-timeout", Input).value = ""
+        self.query_one("#step-retry-count", Input).value = ""
         status = self.query_one("#step-editor-status", Static)
         status.update("")
 
@@ -148,6 +152,7 @@ class StepsEditor(Vertical):
             step.expected, ensure_ascii=False
         )
         self.query_one("#step-timeout", Input).value = str(step.timeout) if step.timeout else ""
+        self.query_one("#step-retry-count", Input).value = str(step.retry_count) if step.retry_count else ""
         status = self.query_one("#step-editor-status", Static)
         status.update(f"Editing Step {idx+1}")
 
@@ -169,7 +174,12 @@ class StepsEditor(Vertical):
         timeout_text = self.query_one("#step-timeout", Input).value.strip()
         timeout = float(timeout_text) if timeout_text else None
 
-        new_step = TestCaseStep(command=cmd, args=args, expected=expected, timeout=timeout)
+        retry_text = self.query_one("#step-retry-count", Input).value.strip()
+        retry_count = int(retry_text) if retry_text else 0
+
+        new_step = TestCaseStep(
+            command=cmd, args=args, expected=expected, timeout=timeout, retry_count=retry_count,
+        )
 
         if self._editing_idx >= 0:
             self._steps[self._editing_idx] = new_step

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2/src/cli_test_framework.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cli-test-framework
-Version: 1.0.1
+Version: 1.0.2
 Summary: A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.
 Home-page: https://github.com/ozil111/cli-test-framework
 Author: Xiaotong Wang

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/src/cli_test_framework.egg-info/SOURCES.txt

@@ -172,6 +172,7 @@ tests/unit/core/__pycache__/test_assertions.cpython-312-pytest-7.4.4.pyc
 tests/unit/core/__pycache__/test_assertions.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/__pycache__/test_config_loader.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/__pycache__/test_execution.cpython-312-pytest-9.0.3.pyc
+tests/unit/core/__pycache__/test_execution.cpython-312.pyc
 tests/unit/core/__pycache__/test_history_store.cpython-312-pytest-7.4.4.pyc
 tests/unit/core/__pycache__/test_history_store.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/__pycache__/test_process_worker.cpython-312-pytest-7.4.4.pyc

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/tests/integration/test_sequence.py

@@ -286,16 +286,21 @@ class TestTestCaseStepDataclass(unittest.TestCase):
     def test_test_case_to_dict_with_steps(self):
         steps = [
             TestCaseStep(command="echo", args=["a"], expected={"return_code": 0}),
+            TestCaseStep(command="echo", args=["b"], expected={"return_code": 0}, retry_count=2),
         ]
-        case = TestCase(name="seq", steps=steps)
+        case = TestCase(name="seq", steps=steps, retry_count=3)
         d = case.to_dict()
         self.assertIn("steps", d)
         self.assertEqual(d["steps"][0]["command"], "echo")
+        self.assertEqual(d["steps"][0]["retry_count"], 0)
+        self.assertEqual(d["steps"][1]["retry_count"], 2)
+        self.assertEqual(d["retry_count"], 3)
 
     def test_test_case_to_dict_without_steps(self):
-        case = TestCase(name="s", command="echo", args=["x"], expected={"return_code": 0})
+        case = TestCase(name="s", command="echo", args=["x"], expected={"return_code": 0}, retry_count=5)
         d = case.to_dict()
         self.assertNotIn("steps", d)
+        self.assertEqual(d["retry_count"], 5)
 
 
 if __name__ == "__main__":

{cli_test_framework-1.0.1 → cli_test_framework-1.0.2}/tests/unit/core/test_config_loader.py

@@ -1,8 +1,8 @@
-"""Unit tests for cli_test_framework.core.config_loader — substitute_placeholders."""
+"""Unit tests for cli_test_framework.core.config_loader — substitute_placeholders, parse_test_cases."""
 
 import pytest
 
-from cli_test_framework.core.config_loader import substitute_placeholders
+from cli_test_framework.core.config_loader import substitute_placeholders, parse_test_cases
 
 
 # ---------------------------------------------------------------------------
@@ -134,3 +134,88 @@ class TestPlaceholderInTestCases:
         assert steps[0]["args"] == ["--parse", "model.dat"]
         assert steps[1]["command"] == "/opt/solver"
         assert steps[1]["args"] == ["--solve", "model.dat"]
+
+
+# ---------------------------------------------------------------------------
+# parse_test_cases — retry_count
+# ---------------------------------------------------------------------------
+
+class TestParseRetryCount:
+    """Tests for ``retry_count`` parsing in ``parse_test_cases``."""
+
+    def test_single_command_retry_count(self):
+        config = {
+            "test_cases": [
+                {
+                    "name": "retry-case",
+                    "command": "echo",
+                    "args": ["ok"],
+                    "expected": {"return_code": 0},
+                    "retry_count": 3,
+                }
+            ]
+        }
+        cases = parse_test_cases(config)
+        assert len(cases) == 1
+        assert cases[0].retry_count == 3
+
+    def test_single_command_default_retry_count(self):
+        """Without ``retry_count``, it defaults to 0."""
+        config = {
+            "test_cases": [
+                {
+                    "name": "no-retry",
+                    "command": "echo",
+                    "args": ["ok"],
+                    "expected": {"return_code": 0},
+                }
+            ]
+        }
+        cases = parse_test_cases(config)
+        assert cases[0].retry_count == 0
+
+    def test_step_retry_count(self):
+        config = {
+            "test_cases": [
+                {
+                    "name": "step-case",
+                    "steps": [
+                        {
+                            "command": "echo",
+                            "args": ["s1"],
+                            "expected": {"return_code": 0},
+                        },
+                        {
+                            "command": "echo",
+                            "args": ["s2"],
+                            "expected": {"return_code": 0},
+                            "retry_count": 2,
+                        },
+                    ],
+                }
+            ]
+        }
+        cases = parse_test_cases(config)
+        assert len(cases) == 1
+        steps = cases[0].steps
+        assert len(steps) == 2
+        assert steps[0].retry_count == 0  # default
+        assert steps[1].retry_count == 2
+
+    def test_step_default_retry_count(self):
+        config = {
+            "test_cases": [
+                {
+                    "name": "step-no-retry",
+                    "steps": [
+                        {
+                            "command": "echo",
+                            "args": ["s1"],
+                            "expected": {"return_code": 0},
+                        }
+                    ],
+                }
+            ]
+        }
+        cases = parse_test_cases(config)
+        assert cases[0].steps[0].retry_count == 0