symtest-cli 1.3.0__py3-none-any.whl

symtest/config/config_schema.py

@@ -0,0 +1,330 @@
+"""JSON Schema (draft 2020-12) for symtest configuration files.
+
+Single source of truth for AI agents (and humans) that *generate* test
+configs.  Exposed via ``symtest schema``.
+
+Keep in sync with:
+- ``core.config_loader.parse_test_cases`` (accepted fields)
+- ``config.config_io.validate_config`` (required fields)
+- ``docs/user_manual.md`` (documented behavior)
+"""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Dict
+
+CONFIG_SCHEMA: Dict[str, Any] = {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "symtest configuration",
+    "description": (
+        "Test suite configuration for the symtest framework. "
+        "YAML files follow the same structure (YAML is a JSON superset)."
+    ),
+    "type": "object",
+    "required": ["test_cases"],
+    "additionalProperties": False,
+    "properties": {
+        "setup": {
+            "type": "object",
+            "additionalProperties": False,
+            "description": "Suite-level setup applied before any test runs.",
+            "properties": {
+                "environment_variables": {
+                    "type": "object",
+                    "description": "Environment variables injected into every test command.",
+                    "additionalProperties": {"type": "string"},
+                },
+            },
+        },
+        "test_cases": {
+            "type": "array",
+            "description": "List of test cases, sequence cases, or import references.",
+            "items": {
+                "anyOf": [
+                    {"$ref": "#/$defs/singleCase"},
+                    {"$ref": "#/$defs/sequenceCase"},
+                    {"$ref": "#/$defs/importRef"},
+                ],
+            },
+        },
+    },
+    "$defs": {
+        "expected": {
+            "type": "object",
+            "additionalProperties": False,
+            "description": (
+                "Assertions evaluated after the command finishes. "
+                "All declared assertions must pass; the combined "
+                "stdout+stderr output is used for output assertions."
+            ),
+            "properties": {
+                "return_code": {
+                    "type": ["integer", "null"],
+                    "description": "Expected process exit code.",
+                },
+                "output_contains": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Strings that must each appear in the command output.",
+                },
+                "output_matches": {
+                    "type": ["string", "null"],
+                    "description": "Regex (re.search) the command output must match.",
+                },
+                "compare_files": {
+                    "type": "array",
+                    "description": "File comparison assertions against golden/baseline files.",
+                    "items": {"$ref": "#/$defs/compareSpec"},
+                },
+            },
+        },
+        "compareSpec": {
+            "type": "object",
+            "required": [],
+            "additionalProperties": True,
+            "description": (
+                "One file comparison rule. actual/baseline are required for built-in "
+                "file-type comparators (text/csv/json/xml/h5/binary). "
+                "They are optional for script or custom (plugin) comparator types. "
+                "Keys other than those listed below are forwarded to the comparator "
+                "as kwargs (e.g. rtol, atol, encoding, tables, data_filter, "
+                "pass_threshold, pass_pattern)."
+            ),
+            "properties": {
+                "actual": {
+                    "type": "string",
+                    "description": "File produced by the test command. Optional when type is 'script' or a workspace plugin.",
+                },
+                "baseline": {
+                    "type": "string",
+                    "description": "Golden/reference file. Optional when type is 'script' or a workspace plugin.",
+                },
+                "type": {
+                    "type": "string",
+                    "description": (
+                        "Comparator type. Built-ins: text, json, csv, xml, h5, binary, script. "
+                        "Custom (workspace plugin) comparator types are also allowed. "
+                        "Omit to auto-detect from the actual file extension."
+                    ),
+                },
+                "start_line": {"type": "integer", "minimum": 1, "description": "Only compare from this line (1-based)."},
+                "end_line": {"type": "integer", "minimum": 1, "description": "Only compare up to this line (1-based)."},
+                "start_column": {"type": "integer", "minimum": 1, "description": "Only compare from this column (1-based)."},
+                "end_column": {"type": "integer", "minimum": 1, "description": "Only compare up to this column (1-based)."},
+                "script": {"type": "string", "description": "Path to the analysis script (script / custom comparator types)."},
+                "case_dir": {"type": "string", "description": "Working directory for the analysis script."},
+                "cwd": {"type": "string", "description": "Working directory for script execution (alias for case_dir)."},
+                "pass_threshold": {"type": "number", "description": "Numeric threshold below which the comparison is considered a pass."},
+                "pass_exit_code": {"type": "integer", "default": 0, "description": "Process exit code that indicates a pass (script comparator)."},
+                "pass_pattern": {"type": "string", "description": "Regex that must match stdout for a pass (script comparator)."},
+                "fail_pattern": {"type": "string", "description": "Regex that, if matched in stdout, forces a fail (script comparator)."},
+                "interpreter": {"type": "string", "description": "Python interpreter to use for running the script."},
+                "timeout": {"type": "number", "description": "Per-comparison timeout in seconds."},
+            },
+        },
+        "resources": {
+            "type": "object",
+            "additionalProperties": False,
+            "description": "Optional scheduling hints (parallel mode).",
+            "properties": {
+                "estimated_time": {"type": "number", "description": "Estimated duration in seconds; used for LPT ordering."},
+                "min_memory_mb": {"type": "number", "description": "Soft memory hint to avoid OOM."},
+                "priority": {"type": "integer", "description": "Higher value => higher priority."},
+                "cpu_cores": {"type": "integer", "minimum": 1, "description": "CPU cores required by this task."},
+            },
+        },
+        "step": {
+            "type": "object",
+            "required": ["command", "args", "expected"],
+            "additionalProperties": False,
+            "properties": {
+                "command": {"type": "string"},
+                "args": {
+                    "type": "array",
+                    "items": {"type": ["string", "number", "boolean"]},
+                },
+                "expected": {"$ref": "#/$defs/expected"},
+                "timeout": {"type": ["number", "null"], "description": "Per-step timeout in seconds."},
+                "retry_count": {"type": "integer", "minimum": 0, "description": "Per-step retry count on failure."},
+            },
+        },
+        "singleCase": {
+            "allOf": [
+                {
+                    "if": {"required": ["extends"]},
+                    "then": {"required": ["name"]},
+                    "else": {"required": ["name", "command", "args", "expected"]},
+                },
+                {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "properties": {
+                        "name": {"type": "string", "description": "Unique test case name."},
+                        "command": {
+                            "type": "string",
+                            "description": (
+                                "Command to execute. May include leading arguments "
+                                "(e.g. 'python ./run.py'); the framework splits and "
+                                "path-resolves them."
+                            ),
+                        },
+                        "args": {
+                            "type": "array",
+                            "items": {"type": ["string", "number", "boolean"]},
+                        },
+                        "expected": {"$ref": "#/$defs/expected"},
+                        "description": {"type": ["string", "null"]},
+                        "timeout": {
+                            "type": ["number", "null"],
+                            "description": "Timeout in seconds (default 3600); null = no limit.",
+                        },
+                        "retry_count": {
+                            "type": "integer",
+                            "minimum": 0,
+                            "description": "Retries after the first failure; passing after retry marks the result flaky.",
+                        },
+                        "tags": {
+                            "type": "array",
+                            "items": {"type": "string"},
+                            "description": "Tags for --tag filtering.",
+                        },
+                        "resources": {"$ref": "#/$defs/resources"},
+                        "expected_failure": {
+                            "type": "boolean",
+                            "description": (
+                                "Mark this case as an expected failure (xfail). "
+                                "When true, a failure counts as XFailed (not a suite failure); "
+                                "an unexpected pass counts as XPassed (suite failure)."
+                            ),
+                        },
+                        "xfail_reason": {
+                            "type": "string",
+                            "description": "Optional reason displayed in the report alongside XFailed results.",
+                        },
+                        "xfail_quiet": {
+                            "type": "boolean",
+                            "description": (
+                                "When true and the case is xfailed (expected failure confirmed), "
+                                "suppress the Command Output block in reports to reduce noise."
+                            ),
+                        },
+                        "abstract": {
+                            "type": "boolean",
+                            "description": (
+                                "When true, this case is a template (base) and is not "
+                                "executed.  Other cases can extend it via 'extends'."
+                            ),
+                        },
+                        "extends": {
+                            "type": "string",
+                            "description": (
+                                "Name of the base test case to inherit from.  Fields "
+                                "from the base are deep-merged; the child's fields take "
+                                "precedence.  Supports chain inheritance with cycle detection."
+                            ),
+                        },
+                        "variables": {
+                            "type": "object",
+                            "additionalProperties": {"type": ["string", "number", "boolean"]},
+                            "description": (
+                                "Per-case placeholder variables for {key} substitution. "
+                                "Merged from the ancestor chain (child overrides parent), "
+                                "then overlaid by global --var flags at run time."
+                            ),
+                        },
+                    },
+                },
+            ],
+        },
+        "sequenceCase": {
+            "allOf": [
+                {
+                    "if": {"required": ["extends"]},
+                    "then": {"required": ["name"]},
+                    "else": {"required": ["name", "steps"]},
+                },
+                {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "description": (
+                        "Multi-step case: steps run in order with fail-fast semantics. "
+                        "The case-level 'expected' is evaluated once after all steps pass."
+                    ),
+                    "properties": {
+                        "name": {"type": "string"},
+                        "steps": {
+                            "type": "array",
+                            "minItems": 1,
+                            "items": {"$ref": "#/$defs/step"},
+                        },
+                        "expected": {
+                            "$ref": "#/$defs/expected",
+                            "description": "Optional case-level assertions (e.g. compare_files on produced files), evaluated after all steps pass.",
+                        },
+                        "description": {"type": ["string", "null"]},
+                        "tags": {"type": "array", "items": {"type": "string"}},
+                        "resources": {"$ref": "#/$defs/resources"},
+                        "expected_failure": {
+                            "type": "boolean",
+                            "description": (
+                                "Mark this case as an expected failure (xfail). "
+                                "When true, a failure counts as XFailed (not a suite failure); "
+                                "an unexpected pass counts as XPassed (suite failure)."
+                            ),
+                        },
+                        "xfail_reason": {
+                            "type": "string",
+                            "description": "Optional reason displayed in the report alongside XFailed results.",
+                        },
+                        "xfail_quiet": {
+                            "type": "boolean",
+                            "description": (
+                                "When true and the case is xfailed (expected failure confirmed), "
+                                "suppress the Command Output block in reports to reduce noise."
+                            ),
+                        },
+                        "abstract": {
+                            "type": "boolean",
+                            "description": (
+                                "When true, this case is a template (base) and is not "
+                                "executed.  Other cases can extend it via 'extends'."
+                            ),
+                        },
+                        "extends": {
+                            "type": "string",
+                            "description": (
+                                "Name of the base test case to inherit from.  Fields "
+                                "from the base are deep-merged; the child's fields take "
+                                "precedence.  Supports chain inheritance with cycle detection."
+                            ),
+                        },
+                        "variables": {
+                            "type": "object",
+                            "additionalProperties": {"type": ["string", "number", "boolean"]},
+                            "description": (
+                                "Per-case placeholder variables for {key} substitution. "
+                                "Merged from the ancestor chain (child overrides parent), "
+                                "then overlaid by global --var flags at run time."
+                            ),
+                        },
+                    },
+                },
+            ],
+        },
+        "importRef": {
+            "type": "object",
+            "required": ["import"],
+            "additionalProperties": False,
+            "description": "Inline another config file (path relative to this file).",
+            "properties": {
+                "import": {"type": "string"},
+            },
+        },
+    },
+}
+
+
+def get_config_schema() -> Dict[str, Any]:
+    """Return a deep copy of the config JSON Schema (safe to mutate)."""
+    return copy.deepcopy(CONFIG_SCHEMA)

symtest/config/import_expander.py

@@ -0,0 +1,149 @@
+"""
+Import reference expander for test configuration files.
+
+Allows a main config file to reference sub-config files via ``"import"``
+entries inside ``test_cases``, recursively expanding them at load time.
+
+Example::
+
+    {
+      "setup": { ... },
+      "test_cases": [
+        { "import": "cases/text_tests.json", "tags": ["text", "fast"] },
+        { "import": "cases/json_tests.yaml" },
+        { "name": "inline_case", "command": "echo", ... }
+      ]
+    }
+
+When ``"tags"`` is specified on an import entry, the tags are injected into
+every test case from that imported file.  Tags already present on individual
+cases are merged (import-level tags come first, deduplicated).
+
+The expansion produces a flat ``test_cases`` list with all imported cases
+inlined.  The Runner layer never sees the ``import`` keys.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+logger = logging.getLogger("symtest.config.import_expander")
+
+
+def _load_raw_config(file_path: Path) -> Dict[str, Any]:
+    """Load a raw config dict from a JSON or YAML file."""
+    ext = file_path.suffix.lower()
+    if ext == ".json":
+        with open(file_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    elif ext in (".yaml", ".yml"):
+        import yaml
+
+        with open(file_path, "r", encoding="utf-8") as f:
+            return yaml.safe_load(f)
+    else:
+        raise ValueError(
+            f"Unsupported config file format: {file_path} (expected .json, .yaml, or .yml)"
+        )
+
+
+def _deep_merge_setup(base: Dict[str, Any], overlay: Dict[str, Any]) -> Dict[str, Any]:
+    """Merge two setup dicts; overlay values win on conflict."""
+    merged = dict(base)
+    for key, value in overlay.items():
+        if (
+            key in merged
+            and isinstance(merged[key], dict)
+            and isinstance(value, dict)
+        ):
+            # Deep merge nested dicts (e.g. environment_variables)
+            merged[key] = _deep_merge_setup(merged[key], value)
+        else:
+            merged[key] = value
+    return merged
+
+
+def expand_imports(
+    config: Dict[str, Any],
+    config_path: Path,
+    loaded_paths: Optional[Set[str]] = None,
+) -> Dict[str, Any]:
+    """Recursively expand ``import`` references in a config dict.
+
+    For each item in ``config["test_cases"]`` that contains an ``"import"``
+    key, loads the referenced file, recursively expands it, and inlines its
+    ``test_cases`` list in place of the ``import`` entry.
+
+    Parameters
+    ----------
+    config:
+        Raw config dict (already parsed from JSON/YAML).
+    config_path:
+        Absolute or resolved path to the config file (used as the base
+        for resolving relative ``import`` paths).
+    loaded_paths:
+        Set of already-loaded canonical paths (for cycle detection).
+
+    Returns
+    -------
+        A new config dict with all ``import`` references expanded.
+    """
+    if loaded_paths is None:
+        loaded_paths = set()
+
+    canonical = str(config_path.resolve())
+    if canonical in loaded_paths:
+        raise RuntimeError(
+            f"Circular import detected: {canonical} is already loaded. "
+            f"Loaded chain: {loaded_paths}"
+        )
+    loaded_paths.add(canonical)
+
+    base_dir = config_path.parent
+    setup = config.get("setup", {})
+    raw_cases: List[Dict[str, Any]] = config.get("test_cases", [])
+    expanded_cases: List[Dict[str, Any]] = []
+
+    for item in raw_cases:
+        if "import" in item:
+            import_rel = item["import"]
+            import_tags = item.get("tags", [])
+            sub_path = (base_dir / import_rel).resolve()
+
+            if not sub_path.exists():
+                raise FileNotFoundError(
+                    f"Imported config file not found: {sub_path} "
+                    f"(referenced from {config_path})"
+                )
+
+            logger.debug("Expanding import: %s -> %s", import_rel, sub_path)
+
+            sub_config = _load_raw_config(sub_path)
+            # Recursively expand (pass loaded_paths copy for cycle detection)
+            sub_config = expand_imports(sub_config, sub_path, loaded_paths)
+
+            # Merge setup from sub-file
+            sub_setup = sub_config.get("setup", {})
+            if sub_setup:
+                setup = _deep_merge_setup(setup, sub_setup)
+
+            sub_cases = sub_config.get("test_cases", [])
+
+            # Inject import-level tags into each imported case
+            if import_tags:
+                for case in sub_cases:
+                    existing = case.get("tags", [])
+                    case["tags"] = list(dict.fromkeys(import_tags + existing))
+
+            expanded_cases.extend(sub_cases)
+        else:
+            expanded_cases.append(item)
+
+    result: Dict[str, Any] = {"test_cases": expanded_cases}
+    if setup:
+        result["setup"] = setup
+
+    return result

symtest/config/inheritance_expander.py

@@ -0,0 +1,197 @@
+"""
+Test-case inheritance resolver.
+
+Resolves ``extends`` references in ``test_cases``, merges field definitions
+via deep-merge (dict) / whole-replace (list), collects ``variables`` for
+per-case placeholder substitution, and removes ``abstract`` (template) cases.
+
+Used after ``expand_imports`` and before ``substitute_placeholders`` /
+``parse_test_cases`` in the config loading pipeline.
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger("symtest.config.inheritance_expander")
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def resolve_inheritance(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Resolve ``extends`` chains, remove ``abstract`` cases, collect ``variables``.
+
+    Parameters
+    ----------
+    config:
+        Config dict already expanded by ``expand_imports``.  Must contain a
+        ``test_cases`` list; may optionally contain ``setup``.
+
+    Returns
+    -------
+        A new config dict where:
+        - every ``abstract: true`` case has been removed,
+        - every ``extends`` reference has been resolved (merged with parent),
+        - each concrete case may carry a ``variables`` dict (deep-merged from
+          ancestor chain).  The ``variables`` are *not* substituted — that is
+          the job of ``apply_variables``.
+
+    Raises
+    ------
+    ValueError
+        If an ``extends`` target is not found among all test-cases (including
+        abstract ones).
+    ValueError
+        If a circular ``extends`` chain is detected.
+    """
+    result = copy.deepcopy(config)
+    raw_cases: List[Dict[str, Any]] = result.get("test_cases", [])
+    if not raw_cases:
+        return result
+
+    # -- Build name → case lookup (includes abstract bases) --
+    all_by_name: Dict[str, Dict[str, Any]] = {}
+    for case in raw_cases:
+        name = case.get("name")
+        if name:
+            all_by_name[name] = case
+
+    # -- Resolve each case --
+    resolved_cases: List[Dict[str, Any]] = []
+    for case in raw_cases:
+        resolved_case = _resolve_one(case, all_by_name, visited=[])
+        # abstract cases are templates — only exclude them from the final list;
+        # they still serve as bases for extends resolution above
+        if not resolved_case.get("abstract"):
+            resolved_case.pop("abstract", None)
+            resolved_cases.append(resolved_case)
+
+    result["test_cases"] = resolved_cases
+    return result
+
+
+def apply_variables(
+    config: Dict[str, Any],
+    global_variables: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Per-case placeholder substitution with global ``--var`` overlay.
+
+    Parameters
+    ----------
+    config:
+        Config dict already processed by ``resolve_inheritance``.  Each
+        concrete case may carry a ``variables`` key.
+    global_variables:
+        Global variables from ``--var`` CLI flag.  These take precedence
+        over case-level ``variables`` (global overlays).
+
+    Returns
+    -------
+        A new config dict with all ``{placeholder}`` replaced.  The
+        ``variables`` key is consumed (removed) from every case.  ``setup``
+        is substituted with global variables only.
+    """
+    from ..core.config_loader import substitute_placeholders
+
+    result = copy.deepcopy(config)
+    gv = global_variables or {}
+
+    # -- Substitute setup with global variables only --
+    if "setup" in result and gv:
+        result["setup"] = substitute_placeholders(result["setup"], gv)
+
+    for case in result.get("test_cases", []):
+        case_vars = case.pop("variables", {})
+        effective = dict(case_vars)
+        effective.update(gv)  # global overlays
+        if effective:
+            _sub_in_place(case, effective, substitute_placeholders)
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+def _deep_merge_dicts(base: dict, overlay: dict) -> dict:
+    """Deep-merge *overlay* into *base*.
+
+    - Both values are dict → recurse.
+    - Otherwise overlay value wins (list / scalar are whole-replaced, not extended).
+    - Returns a new dict (does not mutate inputs).
+    """
+    merged = dict(base)
+    for key, value in overlay.items():
+        if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
+            merged[key] = _deep_merge_dicts(merged[key], value)
+        else:
+            merged[key] = value
+    return merged
+
+
+def _resolve_one(
+    case: Dict[str, Any],
+    all_by_name: Dict[str, Dict[str, Any]],
+    visited: List[str],
+) -> Dict[str, Any]:
+    """Resolve a single case's ``extends`` chain recursively.
+
+    Returns a fully-merged dict (may have ``abstract: true`` — the caller
+    filters abstract cases from the final list).  Abstract cases *can* serve
+    as extends targets; they are only excluded from the output, not from
+    inheritance resolution.
+    """
+    extends_target = case.get("extends")
+    if extends_target is None:
+        # No inheritance — keep as-is
+        result = dict(case)
+        result.pop("extends", None)
+        return result
+
+    # -- Resolve parent chain --
+    name = case.get("name", "<unnamed>")
+    if extends_target not in all_by_name:
+        raise ValueError(
+            f"extends target not found: '{extends_target}' "
+            f"(referenced by case '{name}')"
+        )
+
+    # Cycle detection
+    if extends_target in visited:
+        chain = " -> ".join(visited + [extends_target])
+        raise ValueError(
+            f"Circular extends detected: {chain}"
+        )
+    visited.append(extends_target)
+
+    parent = all_by_name[extends_target]
+    resolved_parent = _resolve_one(parent, all_by_name, visited)
+
+    visited.pop()
+
+    # -- Merge: parent fields → child fields (child wins) --
+    merged = _deep_merge_dicts(resolved_parent, case)
+
+    # Remove internal markers
+    merged.pop("extends", None)
+
+    # abstract: child's own value wins (default False), NOT inherited from parent
+    merged["abstract"] = case.get("abstract", False)
+
+    return merged
+
+
+def _sub_in_place(
+    case: Dict[str, Any],
+    variables: Dict[str, Any],
+    substitute_fn,
+) -> None:
+    """Apply placeholder substitution to *case* in-place via *substitute_fn*."""
+    substituted = substitute_fn(case, variables)
+    case.clear()
+    case.update(substituted)