symtest-cli 1.3.0__py3-none-any.whl

symtest/commands/compare.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+@file compare.py
+@brief Command for comparing files in symtest
+@author Xiaotong Wang
+@date 2024
+"""
+
+import sys
+import os
+import argparse
+import logging
+from pathlib import Path
+from ..file_comparator.factory import ComparatorFactory
+from ..file_comparator.result import ComparisonResult
+
+logger = logging.getLogger("symtest.commands.compare")
+
+
+def parse_arguments():
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(description="Compare two files.")
+    parser.add_argument("file1", help="Path to the first file")
+    parser.add_argument("file2", help="Path to the second file")
+    parser.add_argument("--start-line", type=int, default=1, help="Starting line number (1-based)")
+    parser.add_argument("--end-line", type=int, help="Ending line number (1-based)")
+    parser.add_argument("--start-column", type=int, default=1, help="Starting column number (1-based)")
+    parser.add_argument("--end-column", type=int, help="Ending column number (1-based)")
+    parser.add_argument("--file-type", help="Type of the files to compare", default="auto")
+    parser.add_argument("--encoding", default="utf-8", help="File encoding for text files")
+    parser.add_argument("--chunk-size", type=int, default=8192, help="Chunk size for binary comparison")
+    parser.add_argument("--output-format", choices=["text", "json", "html"], default="text",
+                        help="Output format for the comparison result")
+    parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output")
+    parser.add_argument("--debug", action="store_true", help="Enable debug mode with detailed logging")
+    parser.add_argument("--similarity", action="store_true",
+                        help="When comparing binary files, compute and show similarity index")
+    parser.add_argument("--num-threads", type=int, default=4, help="Number of threads for parallel processing")
+    
+    # CSV comparison options
+    csv_group = parser.add_argument_group('CSV comparison options')
+    csv_group.add_argument("--csv-rtol", type=float, default=1e-5,
+                         help="Relative tolerance for numerical comparison in CSV files")
+    csv_group.add_argument("--csv-atol", type=float, default=1e-8,
+                         help="Absolute tolerance for numerical comparison in CSV files")
+    csv_group.add_argument("--csv-delimiter", default=",",
+                         help="CSV field delimiter (default: comma)")
+    csv_group.add_argument("--csv-quotechar", default='"',
+                         help="Character used for quoting fields in CSV (default: double quote)")
+    csv_group.add_argument("--csv-data-filter", type=str,
+                         help="Data filter to apply before comparison. "
+                              "Example: '>1e-6', '<=0.01', 'abs>1e-9'. "
+                              "Filters out numeric cells that do not meet the criteria "
+                              "from BOTH files before comparison.")
+
+    # JSON comparison options
+    json_group = parser.add_argument_group('JSON comparison options')
+    json_group.add_argument("--json-compare-mode", choices=["exact", "key-based"], default="exact",
+                      help="JSON comparison mode: exact (default) or key-based")
+    json_group.add_argument("--json-key-field", help="Key field(s) to use for key-based JSON comparison")
+    
+    # H5 comparison options
+    h5_group = parser.add_argument_group('HDF5 comparison options')
+    h5_group.add_argument("--h5-table", help="Comma-separated list of table names to compare in HDF5 files")
+    h5_group.add_argument("--h5-table-regex", help="Comma-separated list of regular expression patterns to match table names in HDF5 files. Each pattern is matched independently.")
+    h5_group.add_argument("--h5-structure-only", action="store_true", 
+                         help="Only compare HDF5 file structure without comparing content")
+    h5_group.add_argument("--h5-show-content-diff", action="store_true",
+                         help="Show detailed content differences when content differs")
+    h5_group.add_argument("--h5-rtol", type=float, default=1e-5,
+                         help="Relative tolerance for numerical comparison in HDF5 files")
+    h5_group.add_argument("--h5-atol", type=float, default=1e-8,
+                         help="Absolute tolerance for numerical comparison in HDF5 files")
+    h5_group.add_argument("--h5-data-filter", type=str,
+                         help="Data filter to apply before comparison. "
+                              "Example: '>1e-6', '<=0.01', 'abs>1e-9'. "
+                              "Filters out data that does not meet the criteria from BOTH files before comparison.")
+    h5_group.add_argument("--h5-no-expand-path", dest="h5_expand_path", action="store_false",
+                         help="Do not expand HDF5 group paths to compare all sub-items.")
+    
+    return parser.parse_args()
+
+def detect_file_type(file_path):
+    """Detect the type of file based on its extension"""
+    ext = Path(file_path).suffix.lower()
+    if ext == '.json':
+        return 'json'
+    elif ext == '.xml':
+        return 'xml'
+    elif ext in ['.h5', '.hdf5']:
+        return 'h5'
+    elif ext == '.csv':
+        return 'csv'
+    elif ext in ['.txt', '.py', '.md', '.html', '.css', '.js']:
+        return 'text'
+    else:
+        return 'binary'
+
+def format_result(result, output_format):
+    """Format the comparison result according to the specified output format"""
+    if output_format == "json":
+        return result.to_json()
+    elif output_format == "html":
+        return result.to_html()
+    else:
+        return str(result)
+
+def run_comparison(args, logger=None):
+    """Execute file comparison from a parsed arguments namespace.
+
+    This function can be called from both the standalone compare-files entry point
+    and the symtest compare subcommand.
+
+    Returns:
+        int: Exit code (0 if files are identical, 1 otherwise)
+    """
+    if logger is None:
+        logger = logging.getLogger("symtest.commands.compare")
+
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+        logger.debug("Debug mode enabled")
+
+    # Adjust for 0-based indexing
+    start_line = max(0, args.start_line - 1)
+    end_line = None if args.end_line is None else max(0, args.end_line - 1)
+    start_column = max(0, args.start_column - 1)
+    end_column = None if args.end_column is None else max(0, args.end_column - 1)
+
+    # Resolve file paths
+    file1_path = Path(args.file1).resolve()
+    file2_path = Path(args.file2).resolve()
+
+    if not file1_path.exists():
+        raise ValueError(f"File not found: {file1_path}")
+    if not file2_path.exists():
+        raise ValueError(f"File not found: {file2_path}")
+
+    # Determine file type
+    file_type = args.file_type
+    if file_type == "auto":
+        file_type = detect_file_type(file1_path)
+        logger.info(f"Auto-detected file type: {file_type}")
+
+    # Prepare comparator kwargs
+    comparator_kwargs = {
+        "encoding": args.encoding,
+        "chunk_size": args.chunk_size,
+        "verbose": args.verbose or args.debug,
+        "num_threads": args.num_threads
+    }
+    
+    # Add file type specific arguments
+    if file_type == "json":
+        comparator_kwargs["compare_mode"] = args.json_compare_mode
+        if args.json_key_field:
+            key_fields = [field.strip() for field in args.json_key_field.split(',')]
+            comparator_kwargs["key_field"] = key_fields[0] if len(key_fields) == 1 else key_fields
+    
+    if file_type == "csv":
+        comparator_kwargs["rtol"] = args.csv_rtol
+        comparator_kwargs["atol"] = args.csv_atol
+        comparator_kwargs["delimiter"] = args.csv_delimiter
+        comparator_kwargs["quotechar"] = args.csv_quotechar
+        if args.csv_data_filter:
+            comparator_kwargs["data_filter"] = args.csv_data_filter
+
+    if file_type == "h5":
+        if args.h5_table:
+            tables = [table.strip() for table in args.h5_table.split(',')]
+            comparator_kwargs["tables"] = tables
+        if args.h5_table_regex:
+            comparator_kwargs["table_regex"] = args.h5_table_regex
+        comparator_kwargs["structure_only"] = args.h5_structure_only
+        comparator_kwargs["show_content_diff"] = args.h5_show_content_diff
+        comparator_kwargs["rtol"] = args.h5_rtol
+        comparator_kwargs["atol"] = args.h5_atol
+        if args.h5_data_filter:
+            comparator_kwargs["data_filter"] = args.h5_data_filter
+        comparator_kwargs["expand_path"] = args.h5_expand_path
+    
+    if file_type == "binary":
+        comparator_kwargs["similarity"] = args.similarity
+
+    # Create comparator and perform comparison
+    comparator = ComparatorFactory.create_comparator(file_type, **comparator_kwargs)
+    result = comparator.compare_files(
+        file1_path,
+        file2_path,
+        start_line,
+        end_line,
+        start_column,
+        end_column
+    )
+
+    # Output result
+    output = format_result(result, args.output_format)
+    print(output)
+
+    return 0 if result.identical else 1
+
+
+def main():
+    """Main entry point for the compare-files command"""
+    try:
+        args = parse_arguments()
+        exit_code = run_comparison(args)
+        sys.exit(exit_code)
+
+    except ValueError as ve:
+        logger.error(f"ValueError: {ve}")
+        sys.exit(1)
+    except Exception as e:
+        logger.exception(f"An unexpected error occurred")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

symtest/config/__init__.py

@@ -0,0 +1,7 @@
+"""
+Configuration management module for the CLI Testing Framework.
+
+Provides:
+- import_expander  – recursive expansion of ``import`` references in config files
+- config_io        – unified config read/write + validation
+"""

symtest/config/config_io.py

@@ -0,0 +1,346 @@
+"""
+Unified configuration I/O with import-expansion support.
+
+Provides ``load_config`` and ``save_config`` as the canonical read/write
+entry points for test configuration files, with transparent import expansion.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from .import_expander import expand_imports, _load_raw_config
+from .inheritance_expander import resolve_inheritance
+
+logger = logging.getLogger("symtest.config.config_io")
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def load_config(
+    config_file_path: Union[str, Path],
+    *,
+    expand: bool = True,
+) -> Dict[str, Any]:
+    """Load and (optionally) expand import references in a config file.
+
+    Parameters
+    ----------
+    config_file_path:
+        Path to the config file (JSON or YAML).
+    expand:
+        If ``True`` (default), recursively expand ``import`` references.
+
+    Returns
+    -------
+        A config dict with ``test_cases`` (and optionally ``setup``).
+    """
+    path = Path(config_file_path).resolve()
+
+    if not path.exists():
+        raise FileNotFoundError(f"Config file not found: {path}")
+
+    config = _load_raw_config(path)
+
+    if expand:
+        config = expand_imports(config, path)
+        config = resolve_inheritance(config)
+
+    return config
+
+
+def save_config(
+    config: Dict[str, Any],
+    file_path: Union[str, Path],
+) -> None:
+    """Save a config dict to file, choosing the format by extension.
+
+    Parameters
+    ----------
+    config:
+        Config dict (must contain at least ``test_cases``).
+    file_path:
+        Output path; ``.json`` → JSON, ``.yaml``/``.yml`` → YAML.
+    """
+    path = Path(file_path)
+    ext = path.suffix.lower()
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if ext == ".json":
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(config, f, indent=2, ensure_ascii=False)
+    elif ext in (".yaml", ".yml"):
+        import yaml
+
+        with open(path, "w", encoding="utf-8") as f:
+            yaml.dump(config, f, default_flow_style=False, allow_unicode=True)
+    else:
+        raise ValueError(
+            f"Unsupported output format: {ext} (expected .json, .yaml, or .yml)"
+        )
+
+    logger.info("Config saved to: %s", path)
+
+
+# ---------------------------------------------------------------------------
+# Validation
+# ---------------------------------------------------------------------------
+
+def _validate_required_fields(test_case: Dict[str, Any], index: int, source: str) -> List[str]:
+    """Check required fields for a single test case dict. Returns list of error messages."""
+    errors: List[str] = []
+    prefix = f"[{source}] case #{index}"
+
+    name = test_case.get("name", "<unnamed>")
+    prefix = f"[{source}] case '{name}'"
+
+    if "steps" in test_case:
+        # Sequence mode
+        for si, step in enumerate(test_case["steps"]):
+            for field in ("command", "args", "expected"):
+                if field not in step:
+                    errors.append(f"{prefix} step {si}: missing required field '{field}'")
+    else:
+        # Single-command mode
+        for field in ("name", "command", "args", "expected"):
+            if field not in test_case:
+                errors.append(f"{prefix}: missing required field '{field}'")
+
+    return errors
+
+
+# ---------------------------------------------------------------------------
+# Warning-level checks (do not affect ``valid``)
+# ---------------------------------------------------------------------------
+
+def _command_warning(command: Any, workspace: Path) -> Optional[str]:
+    """Return a warning message if a command is clearly not executable.
+
+    Heuristic only — shell builtins, ``{placeholder}`` commands, and anything
+    resolvable via PATH pass silently.
+    """
+    if not isinstance(command, str) or not command.strip():
+        return "empty command"
+
+    from ..core.execution import _SHELL_BUILTINS
+
+    first = command.strip().split()[0]
+    if "{" in first or "}" in first:
+        return None  # variable placeholder; cannot resolve statically
+    if first.lower() in _SHELL_BUILTINS:
+        return None
+    if Path(first).is_absolute() or "/" in first or "\\" in first:
+        # Path-like command: resolve relative to the workspace
+        candidate = Path(first)
+        if not candidate.is_absolute():
+            candidate = workspace / candidate
+        if not candidate.exists():
+            return f"command not found: '{first}'"
+        return None
+    if shutil.which(first) is None:
+        return f"command not found on PATH: '{first}'"
+    return None
+
+
+def _baseline_warnings(expected: Any, prefix: str, workspace: Path) -> List[str]:
+    """Warn when a compare_files baseline file does not exist.
+
+    ``actual`` files are produced by the test command, so only ``baseline``
+    is checked. Placeholders (``{var}``) are skipped.
+    """
+    warnings: List[str] = []
+    if not isinstance(expected, dict):
+        return warnings
+    compare_specs = expected.get("compare_files") or []
+    for spec in compare_specs:
+        if not isinstance(spec, dict):
+            continue
+        baseline = spec.get("baseline")
+        if not baseline or not isinstance(baseline, str) or "{" in baseline:
+            continue
+        candidate = Path(baseline)
+        if not candidate.is_absolute():
+            candidate = workspace / candidate
+        if not candidate.exists():
+            warnings.append(f"{prefix}: baseline file not found: '{baseline}'")
+    return warnings
+
+
+def _validate_case_warnings(
+    test_case: Dict[str, Any], source: str, workspace: Path,
+) -> List[str]:
+    """Warning-level checks for a single test case dict."""
+    warnings: List[str] = []
+    name = test_case.get("name", "<unnamed>")
+    prefix = f"[{source}] case '{name}'"
+
+    if "steps" in test_case:
+        for si, step in enumerate(test_case.get("steps", [])):
+            if not isinstance(step, dict):
+                continue
+            step_prefix = f"{prefix} step {si}"
+            cmd_warn = _command_warning(step.get("command"), workspace)
+            if cmd_warn:
+                warnings.append(f"{step_prefix}: {cmd_warn}")
+            warnings.extend(
+                _baseline_warnings(step.get("expected"), step_prefix, workspace)
+            )
+        warnings.extend(
+            _baseline_warnings(test_case.get("expected"), prefix, workspace)
+        )
+    else:
+        cmd_warn = _command_warning(test_case.get("command"), workspace)
+        if cmd_warn:
+            warnings.append(f"{prefix}: {cmd_warn}")
+        warnings.extend(
+            _baseline_warnings(test_case.get("expected"), prefix, workspace)
+        )
+
+    return warnings
+
+
+def validate_config(
+    config_file_path: Union[str, Path],
+    workspace: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Validate a test config file (with import expansion) without running tests.
+
+    Checks (errors — affect ``valid``):
+    - JSON/YAML syntax correctness (implicit from successful load)
+    - Required fields on every test case
+    - Import target existence
+    - Circular import detection
+    - extends target existence
+    - Circular extends inheritance detection
+
+    Warning-level checks (reported in ``warnings``, do not affect ``valid``):
+    - ``command`` executability (PATH lookup / path existence)
+    - ``compare_files`` baseline file existence
+
+    Returns a dict with keys ``valid`` (bool), ``errors`` (list),
+    ``warnings`` (list), and ``summary`` (dict with file/case counts).
+    """
+    path = Path(config_file_path).resolve()
+    errors: List[str] = []
+    warnings: List[str] = []
+    files_loaded: List[str] = []
+    total_cases = 0
+    ws = Path(workspace).resolve() if workspace else Path.cwd()
+
+    try:
+        config = _load_raw_config(path)
+    except Exception as exc:
+        return {
+            "valid": False,
+            "errors": [f"Syntax error in {path}: {exc}"],
+            "warnings": [],
+            "summary": {"files": 0, "cases": 0},
+        }
+
+    # Collect all files and cases (walk the import tree)
+    all_cases: List[tuple] = []  # (case_dict, source_path)
+
+    def _walk(current_path: Path, current_config: Dict[str, Any], visited: set) -> None:
+        nonlocal total_cases
+        canonical = str(current_path.resolve())
+        if canonical in visited:
+            errors.append(f"Circular import detected: {canonical}")
+            return
+        visited.add(canonical)
+        files_loaded.append(str(current_path))
+
+        for idx, item in enumerate(current_config.get("test_cases", [])):
+            if "import" in item:
+                sub_path = (current_path.parent / item["import"]).resolve()
+                if not sub_path.exists():
+                    errors.append(
+                        f"Import target not found: {sub_path} "
+                        f"(referenced by {current_path})"
+                    )
+                    continue
+                try:
+                    sub_config = _load_raw_config(sub_path)
+                    _walk(sub_path, sub_config, visited)
+                except Exception as exc:
+                    errors.append(f"Error loading {sub_path}: {exc}")
+            else:
+                is_abstract = item.get("abstract", False)
+                has_extends = "extends" in item
+                source = str(current_path)
+
+                all_cases.append((item, source))
+
+                if is_abstract:
+                    # Abstract cases are templates — skip counting and field validation
+                    continue
+
+                total_cases += 1
+                # extends cases inherit fields from parent; skip required-field checks
+                if not has_extends:
+                    errors.extend(_validate_required_fields(item, idx, source))
+                warnings.extend(_validate_case_warnings(item, source, ws))
+
+    _walk(path, config, set())
+
+    # ── Inheritance validation (extends targets + cycle detection) ──
+    def _validate_inheritance() -> None:
+        """Check extends target existence and circular inheritance chains."""
+        names: Dict[str, Dict[str, Any]] = {}
+        for case_dict, source in all_cases:
+            name = case_dict.get("name")
+            if name:
+                if name in names:
+                    # Duplicate names are ambiguous for extends; flag as error
+                    errors.append(
+                        f"Duplicate case name '{name}' "
+                        f"(found in multiple config files; "
+                        f"extends references must be unambiguous)"
+                    )
+                names[name] = case_dict
+
+        for case_dict, source in all_cases:
+            extends_target = case_dict.get("extends")
+            if not extends_target:
+                continue
+            case_name = case_dict.get("name", "<unnamed>")
+            if extends_target not in names:
+                errors.append(
+                    f"[{source}] case '{case_name}': "
+                    f"extends target '{extends_target}' not found"
+                )
+
+        # Cycle detection: follow extends chain from each case
+        for name, case_dict in names.items():
+            visited_chain: List[str] = []
+            current = name
+            while current in names:
+                target = names[current].get("extends")
+                if not target:
+                    break
+                if target in visited_chain:
+                    chain = " -> ".join(visited_chain + [target])
+                    errors.append(f"Circular extends detected: {chain}")
+                    break
+                visited_chain.append(current)
+                current = target
+
+    _validate_inheritance()
+
+    valid = len(errors) == 0
+    return {
+        "valid": valid,
+        "errors": errors,
+        "warnings": warnings,
+        "summary": {
+            "files": len(files_loaded),
+            "cases": total_cases,
+            "files_loaded": files_loaded,
+        },
+    }