cli-test-framework 0.5.2__tar.gz → 0.5.4__tar.gz

{cli_test_framework-0.5.2/src/cli_test_framework.egg-info → cli_test_framework-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cli-test-framework
-Version: 0.5.2
+Version: 0.5.4
 Summary: A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.
 Home-page: https://github.com/ozil111/cli-test-framework
 Author: Xiaotong Wang
@@ -18,10 +18,9 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: dukpy==0.5.0
-Requires-Dist: h5py<4.0.0,>=3.8.0; python_version < "3.12"
-Requires-Dist: h5py<4.0.0,>=3.10.0; python_version >= "3.12"
-Requires-Dist: numpy<2.0.0,>=1.21.0; python_version < "3.12"
-Requires-Dist: numpy<2.0.0,>=1.26.0; python_version >= "3.12"
+Requires-Dist: h5py<4.0.0,>=3.11.0
+Requires-Dist: numpy>=1.21.0; python_version < "3.12"
+Requires-Dist: numpy>=1.26.0; python_version >= "3.12"
 Requires-Dist: setuptools>=75.8.0
 Requires-Dist: wheel>=0.45.1
 Dynamic: author
@@ -41,6 +40,14 @@ A lightweight automated testing framework for command-line applications. Define
 
 Particularly suited for scientific computing — deep HDF5 support with regex table matching, data filtering, and tolerance-based comparison, making simulation result verification effortless.
 
+## Why this exists
+
+This project was created from a real need: regression testing finite-element software.
+
+In solver-style projects, checking the exit code is not enough. A test often needs to run multiple commands, generate numerical result files, compare HDF5/CSV outputs with tolerances, and make sure old cases do not become slower over time.
+
+CLI Test Framework was built for that workflow.
+
 ## Highlights
 
 - **Parallel Execution** — Multi-thread / multi-process, 3-5x speedup

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/README.md

@@ -4,6 +4,14 @@ A lightweight automated testing framework for command-line applications. Define
 
 Particularly suited for scientific computing — deep HDF5 support with regex table matching, data filtering, and tolerance-based comparison, making simulation result verification effortless.
 
+## Why this exists
+
+This project was created from a real need: regression testing finite-element software.
+
+In solver-style projects, checking the exit code is not enough. A test often needs to run multiple commands, generate numerical result files, compare HDF5/CSV outputs with tolerances, and make sure old cases do not become slower over time.
+
+CLI Test Framework was built for that workflow.
+
 ## Highlights
 
 - **Parallel Execution** — Multi-thread / multi-process, 3-5x speedup

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/setup.py

@@ -8,7 +8,7 @@ with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name="cli-test-framework",
-    version="0.5.2",
+    version="0.5.4",
     author="Xiaotong Wang",
     author_email="xiaotongwang98@gmail.com",
     description="A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.",
@@ -19,10 +19,9 @@ setup(
     package_dir={"": "src"},
     install_requires=[
         "dukpy==0.5.0",
-        "h5py>=3.8.0,<4.0.0; python_version<'3.12'",
-        "h5py>=3.10.0,<4.0.0; python_version>='3.12'",
-        "numpy>=1.21.0,<2.0.0; python_version<'3.12'",
-        "numpy>=1.26.0,<2.0.0; python_version>='3.12'",
+        "h5py>=3.11.0,<4.0.0",
+        "numpy>=1.21.0; python_version<'3.12'",
+        "numpy>=1.26.0; python_version>='3.12'",
         "setuptools>=75.8.0",
         "wheel>=0.45.1"
     ],

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/src/cli_test_framework/commands/compare.py

@@ -49,6 +49,17 @@ def parse_arguments():
                         help="When comparing binary files, compute and show similarity index")
     parser.add_argument("--num-threads", type=int, default=4, help="Number of threads for parallel processing")
     
+    # CSV comparison options
+    csv_group = parser.add_argument_group('CSV comparison options')
+    csv_group.add_argument("--csv-rtol", type=float, default=1e-5,
+                         help="Relative tolerance for numerical comparison in CSV files")
+    csv_group.add_argument("--csv-atol", type=float, default=1e-8,
+                         help="Absolute tolerance for numerical comparison in CSV files")
+    csv_group.add_argument("--csv-delimiter", default=",",
+                         help="CSV field delimiter (default: comma)")
+    csv_group.add_argument("--csv-quotechar", default='"',
+                         help="Character used for quoting fields in CSV (default: double quote)")
+
     # JSON comparison options
     json_group = parser.add_argument_group('JSON comparison options')
     json_group.add_argument("--json-compare-mode", choices=["exact", "key-based"], default="exact",
@@ -148,6 +159,12 @@ def main():
                 key_fields = [field.strip() for field in args.json_key_field.split(',')]
                 comparator_kwargs["key_field"] = key_fields[0] if len(key_fields) == 1 else key_fields
         
+        if file_type == "csv":
+            comparator_kwargs["rtol"] = args.csv_rtol
+            comparator_kwargs["atol"] = args.csv_atol
+            comparator_kwargs["delimiter"] = args.csv_delimiter
+            comparator_kwargs["quotechar"] = args.csv_quotechar
+
         if file_type == "h5":
             if args.h5_table:
                 tables = [table.strip() for table in args.h5_table.split(',')]

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/src/cli_test_framework/core/base_runner.py

@@ -20,7 +20,10 @@ class BaseRunner(ABC):
         self.config_path = self.workspace / config_file
         self.test_cases: List[TestCase] = []
         self.test_case_filter: Optional[List[str]] = test_case_filter
-        self.history_dir = history_dir
+        if history_dir:
+            self.history_dir = str((self.workspace / history_dir).resolve())
+        else:
+            self.history_dir = None
         self.regression_threshold = regression_threshold
         self.results: Dict[str, Any] = {
             "total": 0,

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/src/cli_test_framework/file_comparator/csv_comparator.py

@@ -10,6 +10,7 @@
 
 import csv
 import io
+import math
 from .text_comparator import TextComparator
 from .result import Difference
 
@@ -24,7 +25,7 @@ class CsvComparator(TextComparator):
              - Configurable delimiter and quote character
     """
     
-    def __init__(self, encoding="utf-8", delimiter=",", quotechar='"', chunk_size=8192, verbose=False):
+    def __init__(self, encoding="utf-8", delimiter=",", quotechar='"', chunk_size=8192, verbose=False, rtol=1e-5, atol=1e-8):
         """
         @brief Initialize CSV comparator with configuration
         @param encoding str: File encoding (default: utf-8)
@@ -32,10 +33,14 @@ class CsvComparator(TextComparator):
         @param quotechar str: Character used for quoting fields (default: double quote)
         @param chunk_size int: Size of chunks for reading large files
         @param verbose bool: Enable verbose output
+        @param rtol float: Relative tolerance for numerical comparison (default: 1e-5)
+        @param atol float: Absolute tolerance for numerical comparison (default: 1e-8)
         """
         super().__init__(encoding, chunk_size, verbose)
         self.delimiter = delimiter
         self.quotechar = quotechar
+        self.rtol = rtol
+        self.atol = atol
     
     def read_content(self, file_path, start_line=0, end_line=None, start_column=0, end_column=None):
         """
@@ -116,6 +121,14 @@ class CsvComparator(TextComparator):
             # Compare column values
             for j, (cell1, cell2) in enumerate(zip(row1, row2)):
                 if cell1 != cell2:
+                    # Try numeric tolerance comparison
+                    try:
+                        num1 = float(cell1)
+                        num2 = float(cell2)
+                        if math.isclose(num1, num2, rel_tol=self.rtol, abs_tol=self.atol):
+                            continue  # Within tolerance, treat as equal
+                    except (ValueError, TypeError):
+                        pass  # Non-numeric, fall through to string mismatch
                     differences.append(Difference(
                         position=f"row {i+1}, column {j+1}",
                         expected=cell1,
@@ -137,4 +150,6 @@ class CsvComparator(TextComparator):
                 diff_type=f"more differences not shown"
             ))
         
+        if not differences:
+            return True, []
         return False, differences

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/src/cli_test_framework/file_comparator/factory.py

@@ -77,6 +77,11 @@ class ComparatorFactory:
             json_kwargs = {k: v for k, v in kwargs.items()
                          if k in ['encoding', 'chunk_size', 'verbose', 'compare_mode', 'key_field']}
             return comparator_class(**json_kwargs)
+        elif file_type.lower() == 'csv':
+            # CSV comparator accepts specific parameters
+            csv_kwargs = {k: v for k, v in kwargs.items()
+                         if k in ['encoding', 'delimiter', 'quotechar', 'chunk_size', 'verbose', 'rtol', 'atol']}
+            return comparator_class(**csv_kwargs)
         else:
             # Other comparators only accept basic parameters
             basic_kwargs = {k: v for k, v in kwargs.items()

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4/src/cli_test_framework.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cli-test-framework
-Version: 0.5.2
+Version: 0.5.4
 Summary: A powerful command line testing framework in Python with setup modules, parallel execution, and file comparison capabilities.
 Home-page: https://github.com/ozil111/cli-test-framework
 Author: Xiaotong Wang
@@ -18,10 +18,9 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: dukpy==0.5.0
-Requires-Dist: h5py<4.0.0,>=3.8.0; python_version < "3.12"
-Requires-Dist: h5py<4.0.0,>=3.10.0; python_version >= "3.12"
-Requires-Dist: numpy<2.0.0,>=1.21.0; python_version < "3.12"
-Requires-Dist: numpy<2.0.0,>=1.26.0; python_version >= "3.12"
+Requires-Dist: h5py<4.0.0,>=3.11.0
+Requires-Dist: numpy>=1.21.0; python_version < "3.12"
+Requires-Dist: numpy>=1.26.0; python_version >= "3.12"
 Requires-Dist: setuptools>=75.8.0
 Requires-Dist: wheel>=0.45.1
 Dynamic: author
@@ -41,6 +40,14 @@ A lightweight automated testing framework for command-line applications. Define
 
 Particularly suited for scientific computing — deep HDF5 support with regex table matching, data filtering, and tolerance-based comparison, making simulation result verification effortless.
 
+## Why this exists
+
+This project was created from a real need: regression testing finite-element software.
+
+In solver-style projects, checking the exit code is not enough. A test often needs to run multiple commands, generate numerical result files, compare HDF5/CSV outputs with tolerances, and make sure old cases do not become slower over time.
+
+CLI Test Framework was built for that workflow.
+
 ## Highlights
 
 - **Parallel Execution** — Multi-thread / multi-process, 3-5x speedup

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/src/cli_test_framework.egg-info/SOURCES.txt

@@ -75,7 +75,9 @@ tests/fixtures/test_with_setup.yaml
 tests/integration/test_history_feature.py
 tests/integration/test_real_command_execution.py
 tests/integration/test_sequence.py
+tests/integration/__pycache__/test_history_feature.cpython-312-pytest-7.4.4.pyc
 tests/integration/__pycache__/test_history_feature.cpython-312-pytest-9.0.3.pyc
+tests/integration/__pycache__/test_real_command_execution.cpython-312-pytest-7.4.4.pyc
 tests/integration/__pycache__/test_real_command_execution.cpython-312-pytest-9.0.3.pyc
 tests/integration/__pycache__/test_sequence.cpython-312-pytest-7.4.4.pyc
 tests/integration/__pycache__/test_sequence.cpython-312-pytest-9.0.3.pyc
@@ -88,6 +90,7 @@ tests/integration/file_compare/test_xml_compare.py
 tests/integration/file_compare/__pycache__/test_binary_compare.cpython-312-pytest-7.4.4.pyc
 tests/integration/file_compare/__pycache__/test_binary_compare.cpython-312-pytest-9.0.3.pyc
 tests/integration/file_compare/__pycache__/test_binary_compare.cpython-39-pytest-8.3.4.pyc
+tests/integration/file_compare/__pycache__/test_csv_compare.cpython-312-pytest-7.4.4.pyc
 tests/integration/file_compare/__pycache__/test_csv_compare.cpython-312-pytest-9.0.3.pyc
 tests/integration/file_compare/__pycache__/test_h5_compare.cpython-312-pytest-7.4.4.pyc
 tests/integration/file_compare/__pycache__/test_h5_compare.cpython-312-pytest-9.0.3.pyc
@@ -98,6 +101,7 @@ tests/integration/file_compare/__pycache__/test_json_compare.cpython-39-pytest-8
 tests/integration/file_compare/__pycache__/test_text_compare.cpython-312-pytest-7.4.4.pyc
 tests/integration/file_compare/__pycache__/test_text_compare.cpython-312-pytest-9.0.3.pyc
 tests/integration/file_compare/__pycache__/test_text_compare.cpython-39-pytest-8.3.4.pyc
+tests/integration/file_compare/__pycache__/test_xml_compare.cpython-312-pytest-7.4.4.pyc
 tests/integration/file_compare/__pycache__/test_xml_compare.cpython-312-pytest-9.0.3.pyc
 tests/integration/parallel/test_parallel_runner.py
 tests/integration/parallel/__pycache__/test_parallel_runner.cpython-312-pytest-7.4.4.pyc
@@ -110,15 +114,22 @@ tests/integration/path_handling/__pycache__/test_spaces_in_paths.cpython-39-pyte
 tests/unit/test_cli.py
 tests/unit/test_h5_comparator.py
 tests/unit/test_run_all.py
+tests/unit/__pycache__/test_cli.cpython-312-pytest-7.4.4.pyc
 tests/unit/__pycache__/test_cli.cpython-312-pytest-9.0.3.pyc
+tests/unit/__pycache__/test_h5_comparator.cpython-312-pytest-7.4.4.pyc
 tests/unit/__pycache__/test_h5_comparator.cpython-312-pytest-9.0.3.pyc
+tests/unit/__pycache__/test_run_all.cpython-312-pytest-7.4.4.pyc
 tests/unit/__pycache__/test_run_all.cpython-312-pytest-9.0.3.pyc
 tests/unit/commands/test_compare_command.py
+tests/unit/commands/__pycache__/test_compare_command.cpython-312-pytest-7.4.4.pyc
 tests/unit/commands/__pycache__/test_compare_command.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/test_history_store.py
 tests/unit/core/test_process_worker.py
 tests/unit/core/test_setup.py
+tests/unit/core/__pycache__/test_assertions.cpython-312-pytest-9.0.3.pyc
+tests/unit/core/__pycache__/test_history_store.cpython-312-pytest-7.4.4.pyc
 tests/unit/core/__pycache__/test_history_store.cpython-312-pytest-9.0.3.pyc
+tests/unit/core/__pycache__/test_process_worker.cpython-312-pytest-7.4.4.pyc
 tests/unit/core/__pycache__/test_process_worker.cpython-312-pytest-9.0.3.pyc
 tests/unit/core/__pycache__/test_setup.cpython-312-pytest-7.4.4.pyc
 tests/unit/core/__pycache__/test_setup.cpython-312-pytest-9.0.3.pyc
@@ -131,4 +142,6 @@ tests/unit/runners/__pycache__/test_json_yaml_runner.cpython-39-pytest-8.3.4.pyc
 tests/unit/runners/__pycache__/test_test_case_filter.cpython-312-pytest-7.4.4.pyc
 tests/unit/runners/__pycache__/test_test_case_filter.cpython-312-pytest-9.0.3.pyc
 tests/unit/utils/test_report_generator.py
+tests/unit/utils/__pycache__/test_junit_xml_writer.cpython-312-pytest-9.0.3.pyc
+tests/unit/utils/__pycache__/test_report_generator.cpython-312-pytest-7.4.4.pyc
 tests/unit/utils/__pycache__/test_report_generator.cpython-312-pytest-9.0.3.pyc

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/src/cli_test_framework.egg-info/requires.txt

@@ -1,11 +1,10 @@
 dukpy==0.5.0
+h5py<4.0.0,>=3.11.0
 setuptools>=75.8.0
 wheel>=0.45.1
 
 [:python_version < "3.12"]
-h5py<4.0.0,>=3.8.0
-numpy<2.0.0,>=1.21.0
+numpy>=1.21.0
 
 [:python_version >= "3.12"]
-h5py<4.0.0,>=3.10.0
-numpy<2.0.0,>=1.26.0
+numpy>=1.26.0

{cli_test_framework-0.5.2 → cli_test_framework-0.5.4}/tests/integration/file_compare/test_csv_compare.py

@@ -63,3 +63,49 @@ def test_csv_column_range_limits_scope(tmp_path):
     result = compare_csv(file1, file2, start_column=0, end_column=1)
 
     assert result.identical
+
+
+def test_csv_numeric_tolerance_within_range(tmp_path):
+    """Numeric cells within tolerance should be treated as equal"""
+    file1 = tmp_path / "a.csv"
+    file2 = tmp_path / "b.csv"
+    file1.write_text("id,value\n1,1.000001\n2,3.0\n", encoding="utf-8")
+    file2.write_text("id,value\n1,1.0\n2,3.00001\n", encoding="utf-8")
+
+    result = compare_csv(file1, file2, rtol=1e-5, atol=1e-5)
+
+    assert result.identical
+    assert result.differences == []
+
+
+def test_csv_numeric_tolerance_exceeded(tmp_path):
+    """Numeric cells exceeding tolerance should be reported as mismatch"""
+    file1 = tmp_path / "a.csv"
+    file2 = tmp_path / "b.csv"
+    file1.write_text("id,value\n1,1.0\n", encoding="utf-8")
+    file2.write_text("id,value\n1,2.0\n", encoding="utf-8")
+
+    result = compare_csv(file1, file2, rtol=1e-5, atol=1e-8)
+
+    assert not result.identical
+    diff = result.differences[0]
+    assert diff.diff_type == "cell_mismatch"
+    assert diff.expected == "1.0"
+    assert diff.actual == "2.0"
+
+
+def test_csv_numeric_tolerance_mixed_with_string(tmp_path):
+    """Non-numeric cells should still use string comparison even with tolerance enabled"""
+    file1 = tmp_path / "a.csv"
+    file2 = tmp_path / "b.csv"
+    file1.write_text("id,name,value\n1,Ada,1.0\n", encoding="utf-8")
+    file2.write_text("id,name,value\n1,Grace,1.000001\n", encoding="utf-8")
+
+    result = compare_csv(file1, file2, rtol=1e-5, atol=1e-5)
+
+    assert not result.identical
+    # Only the string mismatch should be reported; numeric value is within tolerance
+    diff_types = [d.diff_type for d in result.differences]
+    assert "cell_mismatch" in diff_types
+    string_diffs = [d for d in result.differences if d.diff_type == "cell_mismatch"]
+    assert any(d.expected == "Ada" and d.actual == "Grace" for d in string_diffs)