cli-test-framework 0.9.0__tar.gz → 0.9.1__tar.gz

{cli_test_framework-0.9.0/src/cli_test_framework.egg-info → cli_test_framework-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cli-test-framework
-Version: 0.9.0
+Version: 0.9.1
 Summary: A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.
 Home-page: https://github.com/ozil111/cli-test-framework
 Author: Xiaotong Wang

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/docs/user_manual.md

@@ -5,6 +5,7 @@
 - [安装](#安装)
 - [测试用例定义](#测试用例定义)
 - [运行测试](#运行测试)
+- [占位符（变量替换）](#占位符变量替换)
 - [标签过滤](#标签过滤)
 - [Setup 模块](#setup-模块)
 - [并行测试](#并行测试)
@@ -245,6 +246,99 @@ for detail in runner.results["details"]:
     print(detail["name"], detail["status"], detail.get("message", ""))
 ```
 
+## 占位符（变量替换）
+
+当同一个配置文件需要在不同环境下使用不同参数（如求解器路径、模型文件路径等）时，可以用 `{变量名}` 占位符编写配置，运行时通过 `--var` 或 `variables` 参数传入实际值。
+
+### 编写含占位符的配置
+
+JSON：
+
+```json
+{
+    "test_cases": [
+        {
+            "name": "求解器测试",
+            "command": "{solver}",
+            "args": ["--input", "{model}", "--output", "{output}"],
+            "expected": { "return_code": 0 }
+        }
+    ]
+}
+```
+
+YAML：
+
+```yaml
+test_cases:
+  - name: 求解器测试
+    command: "{solver}"
+    args: ["--input", "{model}", "--output", "{output}"]
+    expected:
+      return_code: 0
+```
+
+占位符 `{变量名}` 可出现在配置文件的任意字符串值中，包括 `command`、`args`、`name`、`expected.output_contains` 等。支持同一个字符串中使用多个占位符，如 `"{solver} --input {model}"`。
+
+> **安全设计**：只有 `variables` 字典中存在的 key 才会被替换。`{xxx}` 若无匹配不会报错，而是原样保留。因此 `expected.output_matches` 中的正则模式（如 `{2,}`、`\d{4}`）不受影响。
+
+### 用法
+
+#### CLI
+
+```bash
+# 单个变量
+cli-test run test_cases.json --var solver=/opt/solver/bin/solver.exe
+
+# 多个变量
+cli-test run test_cases.json --var solver=/opt/solver/bin/solver.exe --var model=./data/model.dat
+
+# 与并行模式、标签过滤等组合使用
+cli-test run test_cases.json --var solver=solver.exe --parallel --workers 4 --tag smoke
+```
+
+`--var` 格式为 `KEY=VALUE`，可多次使用。等号分隔，key 和 value 两侧的空格会被自动去除。
+
+#### Python API
+
+```python
+from cli_test_framework.runners import JSONRunner, YAMLRunner, ParallelJSONRunner, ParallelYAMLRunner
+
+# 顺序运行
+runner = JSONRunner(
+    config_file="test_cases.json",
+    variables={
+        "solver": "/opt/solver/bin/solver.exe",
+        "model": "./data/model.dat",
+        "output": "./results/output.dat",
+    },
+)
+success = runner.run_tests()
+
+# 并行运行
+runner = ParallelJSONRunner(
+    config_file="test_cases.json",
+    variables={"solver": "/opt/solver/bin/solver.exe"},
+)
+success = runner.run_tests()
+
+# YAML 同样支持
+runner = YAMLRunner(
+    config_file="test_cases.yaml",
+    variables={"solver": "/opt/solver/bin/solver.exe"},
+)
+success = runner.run_tests()
+```
+
+### 适用场景
+
+| 场景 | 示例 |
+|---|---|
+| 不同求解器版本测试 | `--var solver=v1.0/solver.exe` ↔ `--var solver=v2.0/solver.exe` |
+| 不同输入数据 | `--var model=case_1.dat` ↔ `--var model=case_2.dat` |
+| CI/CD 环境适配 | 本地 `/opt/solver.exe`，CI `/runner/solver.exe` |
+| 多平台路径 | Windows `--var solver=C:\solver.exe`，Linux `--var solver=/opt/solver.exe` |
+
 ## 标签过滤
 
 通过标签（tags）可以对测试用例进行分类，并在运行时按标签批量过滤。标签过滤与名称过滤可同时使用，满足 AND 关系（两个条件都必须满足）。

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/setup.py

@@ -8,7 +8,7 @@ with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name="cli-test-framework",
-    version="0.9.0",
+    version="0.9.1",
     author="Xiaotong Wang",
     author_email="xiaotongwang98@gmail.com",
     description="A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.",

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/src/cli_test_framework/cli.py

@@ -22,6 +22,18 @@ from .utils.junit_xml_writer import write_junit_xml
 logger = logging.getLogger("cli_test_framework.cli")
 
 
+def _parse_vars(var_list):
+    """Parse ``['solver=/path', 'model=./m.dat']`` → ``{'solver': '/path', ...}``."""
+    variables = {}
+    for item in var_list or []:
+        if '=' not in item:
+            logger.warning("Ignoring invalid --var '%s' (expected KEY=VALUE)", item)
+            continue
+        key, _, value = item.partition('=')
+        variables[key.strip()] = value.strip()
+    return variables
+
+
 def create_parser():
     """Create and configure the argument parser"""
     parser = argparse.ArgumentParser(
@@ -61,6 +73,11 @@ Examples:
     run_parser.add_argument('--debug', action='store_true', help='Enable debug mode')
     run_parser.add_argument('--junit-xml', dest='junit_xml',
                            help='Write JUnit XML report to the specified file path')
+    run_parser.add_argument('--var', action='append', default=[],
+                           metavar='KEY=VALUE',
+                           help='Set a variable for config placeholder substitution, '
+                                'e.g. --var solver=/path/to/solver '
+                                '(can be used multiple times)')
 
     # ---- Compare command ----
     compare_parser = subparsers.add_parser('compare', help='Compare two files')
@@ -136,6 +153,8 @@ def run_tests(args):
     # construct Namespace objects without the newer arguments.
     history_dir = getattr(args, 'history_dir', None)
     regression_threshold = getattr(args, 'regression_threshold', 1.5)
+    var_list = getattr(args, 'var', [])
+    variables = _parse_vars(var_list)
 
     try:
         if args.parallel:
@@ -150,6 +169,7 @@ def run_tests(args):
                     test_case_tag_filter=args.tag,
                     history_dir=history_dir,
                     regression_threshold=regression_threshold,
+                    variables=variables,
                 )
             elif file_ext in ['.yaml', '.yml']:
                 runner = ParallelYAMLRunner(
@@ -161,6 +181,7 @@ def run_tests(args):
                     test_case_tag_filter=args.tag,
                     history_dir=history_dir,
                     regression_threshold=regression_threshold,
+                    variables=variables,
                 )
             else:
                 logger.error("Unsupported configuration file format for parallel mode: %s", file_ext)
@@ -175,6 +196,7 @@ def run_tests(args):
                     test_case_tag_filter=args.tag,
                     history_dir=history_dir,
                     regression_threshold=regression_threshold,
+                    variables=variables,
                 )
             elif file_ext in ['.yaml', '.yml']:
                 runner = YAMLRunner(
@@ -184,6 +206,7 @@ def run_tests(args):
                     test_case_tag_filter=args.tag,
                     history_dir=history_dir,
                     regression_threshold=regression_threshold,
+                    variables=variables,
                 )
             else:
                 logger.error("Unsupported configuration file format: %s", file_ext)

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/src/cli_test_framework/core/config_loader.py

@@ -11,6 +11,7 @@ Backward-compatible: the runner classes still expose ``load_test_cases()`` and
 from __future__ import annotations
 
 import logging
+import re
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -20,6 +21,40 @@ from ..utils.path_resolver import resolve_paths
 
 logger = logging.getLogger("cli_test_framework.core.config_loader")
 
+# ---------------------------------------------------------------------------
+# Placeholder substitution
+# ---------------------------------------------------------------------------
+
+_PLACEHOLDER_RE = re.compile(r'\{(\w+)\}')
+
+
+def substitute_placeholders(
+    config: Dict[str, Any],
+    variables: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """递归替换 config 中字符串值的 ``{placeholder}`` 占位符。
+
+    只替换 ``variables`` 中存在的 key，未匹配的 ``{xxx}`` 原样保留，
+    不会影响 ``expected.matches`` 等字段中的正则模式（如 ``{2}``）。
+    """
+    if not variables:
+        return config
+
+    def _sub(value: Any) -> Any:
+        if isinstance(value, str):
+            return _PLACEHOLDER_RE.sub(
+                lambda m: str(variables[m.group(1)])
+                if m.group(1) in variables else m.group(0),
+                value,
+            )
+        if isinstance(value, list):
+            return [_sub(item) for item in value]
+        if isinstance(value, dict):
+            return {k: _sub(v) for k, v in value.items()}
+        return value
+
+    return _sub(config)
+
 
 # ---------------------------------------------------------------------------
 # Test-case parsing (loaded dict → list[TestCase])

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/src/cli_test_framework/runners/config_runner.py

@@ -10,7 +10,7 @@ import logging
 from typing import Optional, Dict, Any, Callable, BinaryIO
 
 from ..core.base_runner import BaseRunner
-from ..core.config_loader import parse_test_cases
+from ..core.config_loader import parse_test_cases, substitute_placeholders
 from ..core.test_case import TestCase
 from ..core.execution import execute_single_test_case
 from ..core.types import TestCaseData
@@ -30,9 +30,11 @@ class ConfigRunner(BaseRunner):
     def __init__(self, config_file: str = "test_cases.json",
                  workspace: Optional[str] = None,
                  config_loader: Optional[Callable[[BinaryIO], Dict[str, Any]]] = None,
+                 variables: Optional[Dict[str, Any]] = None,
                  **kwargs):
         super().__init__(config_file, workspace, **kwargs)
         self._config_loader = config_loader
+        self._variables = variables or {}
         # Backward-compatible attribute for tests that patch path_resolver
         self.path_resolver = PathResolver(self.workspace)
 
@@ -46,6 +48,8 @@ class ConfigRunner(BaseRunner):
             with open(self.config_path, 'r', encoding='utf-8') as f:
                 config = self._config_loader(f)
 
+            config = substitute_placeholders(config, self._variables)
+
             self.load_setup_from_config(config)
             self.test_cases = parse_test_cases(
                 config, self.workspace, self.path_resolver,

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/src/cli_test_framework/runners/parallel_config_runner.py

@@ -10,7 +10,7 @@ import logging
 from typing import Optional, Dict, Any, Callable, BinaryIO
 
 from ..core.parallel_runner import ParallelRunner, AtomicSemaphore
-from ..core.config_loader import parse_test_cases, execute_sequence
+from ..core.config_loader import parse_test_cases, execute_sequence, substitute_placeholders
 from ..core.test_case import TestCase
 from ..core.execution import execute_single_test_case
 from ..core.types import TestCaseData
@@ -32,6 +32,7 @@ class ParallelConfigRunner(ParallelRunner):
                  max_workers: Optional[int] = None,
                  execution_mode: str = "thread",
                  config_loader: Optional[Callable[[BinaryIO], Dict[str, Any]]] = None,
+                 variables: Optional[Dict[str, Any]] = None,
                  **kwargs):
         # Auto-detect physical CPU cores (reserve 2 for OS)
         self.total_physical = os.cpu_count() or 4
@@ -43,6 +44,7 @@ class ParallelConfigRunner(ParallelRunner):
         super().__init__(config_file, workspace, max_workers,
                          execution_mode, **kwargs)
         self._config_loader = config_loader
+        self._variables = variables or {}
         # Backward-compatible attribute for tests that patch path_resolver
         self.path_resolver = PathResolver(self.workspace)
 
@@ -121,6 +123,8 @@ class ParallelConfigRunner(ParallelRunner):
             with open(self.config_path, 'r', encoding='utf-8') as f:
                 config = self._config_loader(f)
 
+            config = substitute_placeholders(config, self._variables)
+
             self.load_setup_from_config(config)
             self.test_cases = parse_test_cases(
                 config, self.workspace, self.path_resolver,

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1/src/cli_test_framework.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cli-test-framework
-Version: 0.9.0
+Version: 0.9.1
 Summary: A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.
 Home-page: https://github.com/ozil111/cli-test-framework
 Author: Xiaotong Wang

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/src/cli_test_framework.egg-info/SOURCES.txt

@@ -130,12 +130,14 @@ tests/unit/commands/test_compare_command.py
 tests/unit/commands/__pycache__/test_compare_command.cpython-312-pytest-7.4.4.pyc
 tests/unit/commands/__pycache__/test_compare_command.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/test_assertions.py
+tests/unit/core/test_config_loader.py
 tests/unit/core/test_execution.py
 tests/unit/core/test_history_store.py
 tests/unit/core/test_process_worker.py
 tests/unit/core/test_setup.py
 tests/unit/core/__pycache__/test_assertions.cpython-312-pytest-7.4.4.pyc
 tests/unit/core/__pycache__/test_assertions.cpython-312-pytest-9.0.3.pyc
+tests/unit/core/__pycache__/test_config_loader.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/__pycache__/test_execution.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/__pycache__/test_history_store.cpython-312-pytest-7.4.4.pyc
 tests/unit/core/__pycache__/test_history_store.cpython-312-pytest-9.0.3.pyc
@@ -149,6 +151,7 @@ tests/unit/runners/test_test_case_filter.py
 tests/unit/runners/test_test_case_tag_filter.py
 tests/unit/runners/__pycache__/test_json_yaml_runner.cpython-312-pytest-7.4.4.pyc
 tests/unit/runners/__pycache__/test_json_yaml_runner.cpython-312-pytest-9.0.3.pyc
+tests/unit/runners/__pycache__/test_json_yaml_runner.cpython-312.pyc
 tests/unit/runners/__pycache__/test_json_yaml_runner.cpython-39-pytest-8.3.4.pyc
 tests/unit/runners/__pycache__/test_test_case_filter.cpython-312-pytest-7.4.4.pyc
 tests/unit/runners/__pycache__/test_test_case_filter.cpython-312-pytest-9.0.3.pyc

cli_test_framework-0.9.1/tests/unit/core/test_config_loader.py

@@ -0,0 +1,136 @@
+"""Unit tests for cli_test_framework.core.config_loader — substitute_placeholders."""
+
+import pytest
+
+from cli_test_framework.core.config_loader import substitute_placeholders
+
+
+# ---------------------------------------------------------------------------
+# Basic substitution
+# ---------------------------------------------------------------------------
+
+class TestSubstitutePlaceholders:
+    """Tests for the substitute_placeholders utility."""
+
+    def test_no_variables_returns_unchanged(self):
+        config = {"command": "{solver}", "args": ["--input", "{model}"]}
+        result = substitute_placeholders(config, None)
+        assert result == config
+
+    def test_empty_variables_returns_unchanged(self):
+        config = {"command": "{solver}", "args": ["--input", "{model}"]}
+        result = substitute_placeholders(config, {})
+        assert result == config
+
+    def test_simple_string_substitution(self):
+        config = {"command": "{solver}"}
+        result = substitute_placeholders(config, {"solver": "/usr/bin/solver"})
+        assert result["command"] == "/usr/bin/solver"
+
+    def test_multiple_placeholders_in_one_string(self):
+        config = {"command": "{solver} --input {model}"}
+        result = substitute_placeholders(
+            config, {"solver": "/usr/bin/solver", "model": "./data.dat"},
+        )
+        assert result["command"] == "/usr/bin/solver --input ./data.dat"
+
+    def test_placeholder_in_list(self):
+        config = {"args": ["--solver", "{solver}", "--input", "{model}"]}
+        result = substitute_placeholders(
+            config, {"solver": "/usr/bin/solver", "model": "./data.dat"},
+        )
+        assert result["args"] == ["--solver", "/usr/bin/solver", "--input", "./data.dat"]
+
+    def test_placeholder_in_nested_dict(self):
+        config = {
+            "test_cases": [
+                {
+                    "name": "test-{tag}",
+                    "command": "{solver}",
+                    "args": ["{model}"],
+                    "expected": {"return_code": 0},
+                }
+            ]
+        }
+        result = substitute_placeholders(
+            config, {"solver": "/usr/bin/solver", "model": "./data.dat", "tag": "alpha"},
+        )
+        case = result["test_cases"][0]
+        assert case["name"] == "test-alpha"
+        assert case["command"] == "/usr/bin/solver"
+        assert case["args"] == ["./data.dat"]
+
+    def test_unmatched_placeholder_preserved(self):
+        """Unmatched {xxx} should be kept as-is (safe for regex patterns like {2})."""
+        config = {"expected": {"output_matches": r"\d{2,4}"}}
+        result = substitute_placeholders(config, {"solver": "/bin/solver"})
+        assert result["expected"]["output_matches"] == r"\d{2,4}"
+
+    def test_partial_match_preserved(self):
+        """Only entirely matching keys are substituted; partial matches stay."""
+        config = {"command": "{solver_extra}"}
+        result = substitute_placeholders(config, {"solver": "/bin/solver"})
+        assert result["command"] == "{solver_extra}"
+
+    def test_non_string_values_unchanged(self):
+        config = {"timeout": 60, "priority": 3.5, "enabled": True, "data": None}
+        result = substitute_placeholders(config, {"timeout": "999"})
+        assert result["timeout"] == 60
+        assert result["priority"] == 3.5
+        assert result["enabled"] is True
+        assert result["data"] is None
+
+    def test_original_config_not_mutated(self):
+        original = {"command": "{solver}", "args": ["{model}"]}
+        result = substitute_placeholders(
+            original, {"solver": "/usr/bin/solver", "model": "./data.dat"},
+        )
+        # Original should be unchanged
+        assert original["command"] == "{solver}"
+        assert original["args"] == ["{model}"]
+        # Result should have substitutions
+        assert result["command"] == "/usr/bin/solver"
+        assert result["args"] == ["./data.dat"]
+
+    def test_variable_value_converted_to_str(self):
+        """Non-string variable values should be converted via str()."""
+        config = {"command": "{count} {flag}"}
+        result = substitute_placeholders(config, {"count": 42, "flag": True})
+        assert result["command"] == "42 True"
+
+
+# ---------------------------------------------------------------------------
+# Integration-style: full config dict with placeholders
+# ---------------------------------------------------------------------------
+
+class TestPlaceholderInTestCases:
+    """End-to-end style tests with realistic config dicts."""
+
+    def test_full_json_config_with_steps(self):
+        config = {
+            "test_cases": [
+                {
+                    "name": "Multi-step with placeholders",
+                    "steps": [
+                        {
+                            "command": "{solver}",
+                            "args": ["--parse", "{input}"],
+                            "expected": {"return_code": 0},
+                        },
+                        {
+                            "command": "{solver}",
+                            "args": ["--solve", "{input}"],
+                            "expected": {"output_contains": ["Optimal"]},
+                        },
+                    ],
+                }
+            ]
+        }
+        result = substitute_placeholders(
+            config, {"solver": "/opt/solver", "input": "model.dat"},
+        )
+        steps = result["test_cases"][0]["steps"]
+        assert steps[0]["command"] == "/opt/solver"
+        assert steps[0]["args"] == ["--parse", "model.dat"]
+        assert steps[1]["command"] == "/opt/solver"
+        assert steps[1]["args"] == ["--solve", "model.dat"]

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/tests/unit/runners/test_json_yaml_runner.py

@@ -82,6 +82,56 @@ class TestYAMLRunner(unittest.TestCase):
         self.temp_dir.cleanup()
 
 
+class TestPlaceholderSubstitution(unittest.TestCase):
+    """Integration test: verify that variables flow through runner's load_test_cases."""
+
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        config = {
+            "test_cases": [
+                {
+                    "name": "placeholder-test",
+                    "command": "{solver}",
+                    "args": ["--input", "{flag}"],
+                    "expected": {"return_code": 0},
+                }
+            ]
+        }
+        self.config_path = Path(self.temp_dir.name) / "placeholder_config.json"
+        self.config_path.write_text(json.dumps(config), encoding="utf-8")
+
+    def test_placeholders_substituted_in_loaded_cases(self):
+        runner = JSONRunner(
+            str(self.config_path),
+            workspace=self.temp_dir.name,
+            variables={"solver": "/usr/bin/solver", "flag": "--verbose"},
+        )
+        runner.load_test_cases()
+
+        self.assertEqual(len(runner.test_cases), 1)
+        case = runner.test_cases[0]
+        self.assertIn("/usr/bin/solver", case.command)
+        self.assertIn("--verbose", case.args)
+        self.assertNotIn("{solver}", case.command)
+        self.assertNotIn("{flag}", case.args)
+
+    def test_no_variables_still_works(self):
+        """Without variables, placeholders should remain as-is."""
+        runner = JSONRunner(
+            str(self.config_path),
+            workspace=self.temp_dir.name,
+        )
+        runner.load_test_cases()
+
+        self.assertEqual(len(runner.test_cases), 1)
+        case = runner.test_cases[0]
+        self.assertIn("{solver}", case.command)
+        self.assertIn("{flag}", case.args)
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
+
 if __name__ == "__main__":
     unittest.main()
 

{cli_test_framework-0.9.0 → cli_test_framework-0.9.1}/tests/unit/test_cli.py

@@ -109,3 +109,49 @@ def test_main_exits_nonzero_without_command(monkeypatch):
 
     assert exc.value.code == 1
 
+
+def test_parse_vars_single():
+    result = cli._parse_vars(["solver=/opt/solver"])
+    assert result == {"solver": "/opt/solver"}
+
+
+def test_parse_vars_multiple():
+    result = cli._parse_vars(["solver=/opt/solver", "model=./data.dat"])
+    assert result == {"solver": "/opt/solver", "model": "./data.dat"}
+
+
+def test_parse_vars_empty():
+    result = cli._parse_vars([])
+    assert result == {}
+
+
+def test_parse_vars_invalid_warns(caplog):
+    result = cli._parse_vars(["bad_entry"])
+    assert result == {}
+    assert "Ignoring invalid --var" in caplog.text
+
+
+def test_parse_vars_key_with_spaces():
+    result = cli._parse_vars([" solver = /opt/solver "])
+    assert result == {"solver": "/opt/solver"}
+
+
+def test_run_tests_passes_variables_to_runner(tmp_path, monkeypatch):
+    config = tmp_path / "cases.json"
+    config.write_text('{"test_cases": []}', encoding="utf-8")
+    captured = {}
+
+    class VarRunner(DummyRunner):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            captured.update(kwargs)
+
+        def run_tests(self):
+            return True
+
+    monkeypatch.setattr(cli, "JSONRunner", VarRunner)
+
+    cli.run_tests(make_args(config, var=["solver=/opt/solver"]))
+
+    assert captured.get("variables") == {"solver": "/opt/solver"}
+