vnnlib-test-solver 2.0.0__py3-none-any.whl

vnnlib_test_solver/validation.py

@@ -0,0 +1,495 @@
+"""Schema validation for a parsed configuration file.
+
+Deliberately a separate pass from loading: TOML syntax and configuration meaning fail
+for different reasons and must produce different messages. Every message names the
+offending key and, where a value comes from a closed set, the legal set.
+
+Values drawn from a closed set need no shape checking, because membership settles it.
+Values a configuration author writes freely do: the response formats in section 5.4 are
+line- and space-delimited, so a stray newline or space inside a value silently changes
+how many answers a consumer reads. Those checks live in ``_reject_line_break`` and
+``_reject_whitespace``, and each says what the consumer would otherwise see.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .errors import ConfigError
+from .injection import KEYS as INJECTION_KEYS
+from .spec import (
+    ARITHMETIC_COMPLEXITY_THEORIES,
+    ELEMENT_TYPES,
+    HIDDEN_NODE_THEORIES,
+    MULTIPLE_INPUT_OUTPUT_THEORIES,
+    MULTIPLE_NETWORK_THEORIES,
+    MULTIPLE_NODE_COMPARISON_THEORIES,
+    REAL,
+    RESULTS,
+    as_list,
+)
+
+TOP_LEVEL_KEYS = frozenset({"solver", "capabilities", "soundness", "rules", "injection"})
+SOLVER_KEYS = frozenset({"name", "version"})
+SOUNDNESS_KEYS = frozenset({"sound-for"})
+RULE_KEYS = (
+    frozenset({"match", "result", "assignments", "model_element_types", "delay_seconds"})
+    | INJECTION_KEYS
+)
+"""The keys a ``[[rules]]`` entry may carry.
+
+Built from the injection key set rather than repeating its four names, so the
+per-rule schema and the ``[injection]`` schema cannot come to disagree about
+which controls exist. ``delay_seconds`` is deliberately not one of them: it is
+about how a deadline is met, which only ``verify`` has.
+"""
+OPERATOR_KEYS = frozenset({"name", "element-types"})
+
+MAX_EXIT_CODE = 255
+"""The largest exit status a configuration may ask for.
+
+A process exit status is not an arbitrary integer. On a Unix-like system it
+carries **eight bits**: the parent reads it through ``WEXITSTATUS``, so 256
+arrives as 0 and a configuration asking a run to fail would report that it
+succeeded. Windows carries the full 32 bits, which is why this is invisible on
+a Windows machine - measured, not assumed, and measured only because the
+consuming process runner is POSIX-only and Windows is where this was written.
+
+Bounded to the range every platform can represent, for the reason
+``MAX_WAIT_SECONDS`` is bounded below the platform's own ceiling: a
+configuration that means one thing on one machine and the opposite on another
+is not a baseline. Refusing a value out of range turns a silent lie into a
+message.
+"""
+
+MAX_WAIT_SECONDS = 2**31 - 1
+"""The longest wait this package will accept, in seconds - about 68 years.
+
+``time.sleep`` raises ``OverflowError`` above roughly 2**63 nanoseconds, and the
+exact ceiling is a property of the platform's clock rather than of anything the
+standard says. Picking a fixed bound well below it keeps the refusal identical on
+every machine, which an inherited platform limit would not: a configuration that
+loads on one machine and raises on another is not a baseline.
+
+It lives here rather than with the code that waits because the configuration's
+``delay_seconds`` and the command line's ``--timeout`` both need it, and this is
+the module both can import without a cycle.
+"""
+
+THEORY_KEYS = {
+    "hidden-node-theories": HIDDEN_NODE_THEORIES,
+    "multiple-input-output-theories": MULTIPLE_INPUT_OUTPUT_THEORIES,
+    "multiple-network-theories": MULTIPLE_NETWORK_THEORIES,
+    "multiple-node-comparison-theories": MULTIPLE_NODE_COMPARISON_THEORIES,
+    "arithmetic-complexity-theories": ARITHMETIC_COMPLEXITY_THEORIES,
+}
+VERSION_PAIR_KEYS = ("onnx-opset-versions", "vnnlib-versions")
+BOOLEAN_KEYS = ("optimised-disjunctive-reasoning", "serialise-assignments")
+CAPABILITY_KEYS = (
+    frozenset(THEORY_KEYS)
+    | frozenset(VERSION_PAIR_KEYS)
+    | frozenset(BOOLEAN_KEYS)
+    | frozenset({"onnx-element-types", "onnx-operators"})
+)
+
+
+def validate(raw: Mapping[str, Any]) -> None:
+    """Raise ConfigError on the first schema violation found."""
+    _reject_unknown_keys(raw, TOP_LEVEL_KEYS, "top level")
+    _validate_solver(raw)
+    _validate_capabilities(raw)
+    _validate_soundness(raw)
+    _validate_injection(raw)
+    _validate_rules(raw)
+
+
+def soundness_warnings(
+    capabilities: Mapping[str, Any], soundness: Mapping[str, Any]
+) -> tuple[str, ...]:
+    """Report element types reported without a matching claim of sound analysis.
+
+    Section 5.4.1 asks a solver to report an element type only where there is
+    strong reason to believe its analysis is sound for that type, and to report
+    only ``real`` otherwise. This is a warning and not an error, because a
+    configuration claiming what it cannot soundly analyse is exactly the
+    condition callers need to be able to trigger on purpose.
+
+    **``real`` is exempt, and reporting it is never warned about.** It is not one
+    claim among the twenty-one: it is what section 5.4.1 tells a solver to report
+    *instead* of the others when it cannot promise sound reasoning over floating
+    point at all. A configuration reporting ``real`` alone and claiming soundness
+    for nothing is doing exactly what the clause asks, so warning at it would be
+    warning about the safe choice - and the noise would fall on precisely the
+    configurations that were most careful.
+
+    The reasoning behind that, which is the part worth not re-deriving: networks
+    run over floating point, floating point is far harder to reason about than
+    real arithmetic, and many solvers simply assume real arithmetic instead. That
+    assumption is unsound and can return a wrong answer on a floating-point edge
+    case, so the standard added ``real`` as the way a solver declares it is doing
+    that. A solver that cannot reason soundly over floating point should report
+    ``real`` and nothing else - and must not report ``float32`` or ``float64``,
+    which is the case this warning exists to catch.
+
+    Returned rather than printed, so the validation pass stays free of output and
+    can be tested without capturing a stream. Order follows the configuration, so
+    the message is identical between runs.
+    """
+    reported = capabilities.get("onnx-element-types", [])
+    claimed = set(soundness.get("sound-for", []))
+    unclaimed = [name for name in reported if name != REAL and name not in claimed]
+    if not unclaimed:
+        return ()
+    return (
+        f"[capabilities] onnx-element-types reports {', '.join(unclaimed)}, which "
+        f"[soundness] sound-for does not claim; section 5.4.1 asks a solver to "
+        f"report only element types it can analyse soundly",
+    )
+
+
+def _reject_unknown_keys(
+    table: Mapping[str, Any], permitted: frozenset[str], where: str
+) -> None:
+    unknown = sorted(set(table) - permitted)
+    if unknown:
+        raise ConfigError(
+            f"unknown configuration key {unknown[0]!r} at {where}; "
+            f"permitted keys are {as_list(permitted)}"
+        )
+
+
+def _require_table(raw: Mapping[str, Any], key: str) -> Mapping[str, Any]:
+    value = raw.get(key)
+    if value is None:
+        raise ConfigError(f"missing required configuration table [{key}]")
+    if not isinstance(value, dict):
+        raise ConfigError(f"[{key}] must be a table, not {_kind(value)}")
+    return value
+
+
+def _validate_solver(raw: Mapping[str, Any]) -> None:
+    solver = _require_table(raw, "solver")
+    _reject_unknown_keys(solver, SOLVER_KEYS, "[solver]")
+    for key in sorted(SOLVER_KEYS):
+        if key not in solver:
+            raise ConfigError(f"[solver] is missing required key {key!r}")
+        if not isinstance(solver[key], str):
+            raise ConfigError(
+                f"[solver] {key} must be a string, not {_kind(solver[key])}"
+            )
+        if not solver[key]:
+            raise ConfigError(f"[solver] {key} must not be empty")
+        # Spaces are allowed here and nowhere else: section 5.2 says the name may
+        # contain them. A line break is still refused, because the answer is one line.
+        _reject_line_break(solver[key], f"[solver] {key}")
+
+
+def _validate_capabilities(raw: Mapping[str, Any]) -> None:
+    if "capabilities" not in raw:
+        return
+    capabilities = _require_table(raw, "capabilities")
+    _reject_unknown_keys(capabilities, CAPABILITY_KEYS, "[capabilities]")
+
+    for key, legal in sorted(THEORY_KEYS.items()):
+        for identifier in _string_list(capabilities, key, "[capabilities]"):
+            if identifier not in legal:
+                raise ConfigError(
+                    f"[capabilities] {key} contains unknown theory identifier "
+                    f"{identifier!r}; the standard defines {as_list(legal)}"
+                )
+
+    for element_type in _string_list(
+        capabilities, "onnx-element-types", "[capabilities]"
+    ):
+        if element_type not in ELEMENT_TYPES:
+            raise ConfigError(
+                f"[capabilities] onnx-element-types contains unknown element type "
+                f"{element_type!r}; the standard defines {as_list(ELEMENT_TYPES)}"
+            )
+
+    for key in VERSION_PAIR_KEYS:
+        versions = _string_list(capabilities, key, "[capabilities]")
+        if key in capabilities and len(versions) != 2:
+            raise ConfigError(
+                f"[capabilities] {key} must be exactly two version strings, "
+                f"minimum then maximum; found {len(versions)}"
+            )
+        for version in versions:
+            if not version:
+                raise ConfigError(
+                    f"[capabilities] {key} must not contain an empty version; the "
+                    f"response is two lines, and an empty one is a blank line that "
+                    f"a consumer reads as a malformed response rather than as a "
+                    f"version"
+                )
+            # A version identifier is one whole line of the response, so anything
+            # that splits or pads a line changes what the consumer reads back.
+            _reject_whitespace(version, f"[capabilities] {key}")
+
+    for key in BOOLEAN_KEYS:
+        if key in capabilities and not isinstance(capabilities[key], bool):
+            raise ConfigError(
+                f"[capabilities] {key} must be true or false, "
+                f"not {_kind(capabilities[key])}"
+            )
+
+    _validate_operators(capabilities)
+
+
+def _validate_operators(capabilities: Mapping[str, Any]) -> None:
+    operators = capabilities.get("onnx-operators", [])
+    if not isinstance(operators, list):
+        raise ConfigError(
+            f"[capabilities] onnx-operators must be an array of tables, "
+            f"not {_kind(operators)}"
+        )
+    for index, operator in enumerate(operators):
+        where = f"[[capabilities.onnx-operators]] entry {index + 1}"
+        if not isinstance(operator, dict):
+            raise ConfigError(f"{where} must be a table, not {_kind(operator)}")
+        _reject_unknown_keys(operator, OPERATOR_KEYS, where)
+        name = operator.get("name")
+        if not isinstance(name, str) or not name:
+            raise ConfigError(f"{where} is missing a non-empty 'name'")
+        # The response puts the operator name and its element types on one line,
+        # separated by spaces. Whitespace inside the name is therefore not a
+        # cosmetic problem: it is read back as a name plus an extra element type,
+        # or as two operators, and nothing anywhere reports an error.
+        _reject_whitespace(name, f"{where} name")
+        for element_type in _string_list(operator, "element-types", where):
+            if element_type not in ELEMENT_TYPES:
+                raise ConfigError(
+                    f"{where} lists unknown element type {element_type!r} for operator "
+                    f"{name!r}; the standard defines {as_list(ELEMENT_TYPES)}"
+                )
+
+
+def _validate_soundness(raw: Mapping[str, Any]) -> None:
+    if "soundness" not in raw:
+        return
+    soundness = _require_table(raw, "soundness")
+    _reject_unknown_keys(soundness, SOUNDNESS_KEYS, "[soundness]")
+    for element_type in _string_list(soundness, "sound-for", "[soundness]"):
+        if element_type not in ELEMENT_TYPES:
+            raise ConfigError(
+                f"[soundness] sound-for contains unknown element type "
+                f"{element_type!r}; the standard defines {as_list(ELEMENT_TYPES)}"
+            )
+
+
+def _validate_injection(raw: Mapping[str, Any]) -> None:
+    """Check the top-level table that misbehaves whichever command runs.
+
+    The same four controls a rule carries, checked by the same function, because
+    two copies of one check are two things that can come to disagree about what
+    a valid configuration is.
+    """
+    if "injection" not in raw:
+        return
+    table = _require_table(raw, "injection")
+    _reject_unknown_keys(table, INJECTION_KEYS, "[injection]")
+    _validate_injection_keys(table, "[injection]")
+
+
+def _validate_rules(raw: Mapping[str, Any]) -> None:
+    rules = raw.get("rules", [])
+    if not isinstance(rules, list):
+        raise ConfigError(f"[[rules]] must be an array of tables, not {_kind(rules)}")
+    for index, rule in enumerate(rules):
+        where = f"[[rules]] entry {index + 1}"
+        if not isinstance(rule, dict):
+            raise ConfigError(f"{where} must be a table, not {_kind(rule)}")
+        _reject_unknown_keys(rule, RULE_KEYS, where)
+
+        match = rule.get("match")
+        if not isinstance(match, str) or not match:
+            raise ConfigError(f"{where} is missing a non-empty 'match' pattern")
+
+        result = rule.get("result")
+        if result is None:
+            raise ConfigError(f"{where} is missing a 'result'")
+        if result not in RESULTS:
+            raise ConfigError(
+                f"{where} has unknown result {result!r}; the standard defines "
+                f"{', '.join(RESULTS)}"
+            )
+
+        _validate_rule_injection(rule, where)
+        _validate_assignments(rule, where)
+        _validate_model_element_types(rule, where)
+
+
+def _validate_rule_injection(rule: Mapping[str, Any], where: str) -> None:
+    _validate_injection_keys(rule, where)
+    if "delay_seconds" in rule and not _is_integer(rule["delay_seconds"]):
+        raise ConfigError(
+            f"{where} delay_seconds must be an integer, not "
+            f"{_kind(rule['delay_seconds'])}"
+        )
+    if rule.get("delay_seconds", 0) < 0:
+        raise ConfigError(f"{where} delay_seconds must not be negative")
+    # An unbounded delay is not merely a long wait: above the platform's clock
+    # limit the sleep raises, and an unhandled exception is a traceback on stderr
+    # rather than a message. Refused here, where it can still be explained.
+    if rule.get("delay_seconds", 0) > MAX_WAIT_SECONDS:
+        raise ConfigError(
+            f"{where} delay_seconds must be at most {MAX_WAIT_SECONDS}; "
+            f"{rule['delay_seconds']} is longer than this package can wait"
+        )
+
+
+def _validate_injection_keys(table: Mapping[str, Any], where: str) -> None:
+    """Check the four deliberate-misbehaviour controls, wherever they appear.
+
+    ``stderr`` and ``raw_stdout`` are exempt from the shape checks every other
+    string value goes through. Producing output a consumer cannot parse is the
+    entire purpose of these two, so refusing a line break inside one would
+    refuse the feature.
+    """
+    if "exit_code" in table:
+        if not _is_integer(table["exit_code"]):
+            raise ConfigError(
+                f"{where} exit_code must be an integer, not {_kind(table['exit_code'])}"
+            )
+        if not 0 <= table["exit_code"] <= MAX_EXIT_CODE:
+            raise ConfigError(
+                f"{where} exit_code must be between 0 and {MAX_EXIT_CODE}; "
+                f"{table['exit_code']} is outside what a process exit status can "
+                f"carry, and on a Unix-like system it would arrive at the caller "
+                f"with its low eight bits taken - {table['exit_code'] & 0xFF} in "
+                f"this case"
+            )
+    for key in ("stderr", "raw_stdout"):
+        if key in table and not isinstance(table[key], str):
+            raise ConfigError(f"{where} {key} must be a string, not {_kind(table[key])}")
+    if "crash" in table and not isinstance(table["crash"], bool):
+        raise ConfigError(
+            f"{where} crash must be true or false, not {_kind(table['crash'])}"
+        )
+    # A process killed by a signal carries no exit status, so a configuration
+    # asking for both is asking for two things only one of which can happen.
+    # Refused rather than silently resolved: honouring one would leave the other
+    # a key that was accepted, validated and then did nothing.
+    if table.get("crash") and "exit_code" in table:
+        raise ConfigError(
+            f"{where} sets both crash and exit_code, and only one of them can "
+            f"happen: a process that terminates abnormally reports no exit status "
+            f"at all. Set one or the other"
+        )
+
+
+def _validate_assignments(rule: Mapping[str, Any], where: str) -> None:
+    assignments = rule.get("assignments", {})
+    if not isinstance(assignments, dict):
+        raise ConfigError(
+            f"{where} assignments must be a table keyed by variable name, "
+            f"not {_kind(assignments)}"
+        )
+    for variable, values in sorted(assignments.items()):
+        if not isinstance(values, list):
+            raise ConfigError(
+                f"{where} assignment for {variable!r} must be an array of values, "
+                f"not {_kind(values)}"
+            )
+        if not values:
+            raise ConfigError(f"{where} assignment for {variable!r} is empty")
+        for value in values:
+            if not _is_integer(value) and not isinstance(value, float):
+                raise ConfigError(
+                    f"{where} assignment for {variable!r} contains a non-numeric "
+                    f"value {value!r}"
+                )
+
+
+def _validate_model_element_types(rule: Mapping[str, Any], where: str) -> None:
+    """Check the types a rule claims the supplied model files expose.
+
+    This package performs no ONNX parsing and never opens a model file, so
+    section 5.3.2's second error condition has nothing to read the model's
+    types out of. The configuration states them instead, keyed by the variable
+    they belong to, and the query supplies the declared type they are compared
+    against. That is the same split the assignment values use: configuration
+    claims, the query decides what the claim is measured against, and neither
+    can silently agree with the other about something the query does not say.
+
+    ``real`` is accepted here even though no ONNX model can expose it. It is
+    not a type a model has, so a configuration naming it is describing
+    something that cannot happen - but section 5.3.2's closing paragraph
+    leaves input the standard does not cover to the solver's discretion, and
+    refusing this would be inventing a rule to enforce rather than reading one.
+    A mismatch is a mismatch whichever of the twenty-one names is on each side.
+    """
+    types = rule.get("model_element_types", {})
+    if not isinstance(types, dict):
+        raise ConfigError(
+            f"{where} model_element_types must be a table keyed by variable name, "
+            f"not {_kind(types)}"
+        )
+    for variable, element_type in sorted(types.items()):
+        if not variable:
+            raise ConfigError(
+                f"{where} model_element_types has an entry with an empty variable name"
+            )
+        if not isinstance(element_type, str):
+            raise ConfigError(
+                f"{where} model_element_types gives {variable!r} a "
+                f"{_kind(element_type)} rather than an element type name"
+            )
+        if element_type not in ELEMENT_TYPES:
+            raise ConfigError(
+                f"{where} model_element_types gives {variable!r} unknown element type "
+                f"{element_type!r}; the standard defines {as_list(ELEMENT_TYPES)}"
+            )
+
+
+def _string_list(table: Mapping[str, Any], key: str, where: str) -> list[str]:
+    value = table.get(key, [])
+    if not isinstance(value, list):
+        raise ConfigError(
+            f"{where} {key} must be an array of strings, not {_kind(value)}"
+        )
+    for entry in value:
+        if not isinstance(entry, str):
+            raise ConfigError(
+                f"{where} {key} must contain only strings; found {_kind(entry)}"
+            )
+    return value
+
+
+def _reject_line_break(value: str, what: str) -> None:
+    """Refuse a value that would turn one line of output into several.
+
+    Every response format in sections 5.3 and 5.4 is line-delimited, so a newline
+    inside a value is not a formatting nicety: one answer arrives as two, and the
+    consumer has no way to tell that is what happened.
+    """
+    if "\n" in value or "\r" in value:
+        raise ConfigError(
+            f"{what} must not contain a line break; it is emitted as one line, and "
+            f"an embedded newline would be read back as more than one answer"
+        )
+
+
+def _reject_whitespace(value: str, what: str) -> None:
+    """Refuse a value that is emitted as a single whitespace-delimited token."""
+    _reject_line_break(value, what)
+    if any(character.isspace() for character in value):
+        raise ConfigError(
+            f"{what} must not contain whitespace: {value!r} would be read back as "
+            f"more than one value, because the response separates values with spaces"
+        )
+
+
+def _is_integer(value: Any) -> bool:
+    """Report whether a value is a TOML integer, excluding booleans.
+
+    ``bool`` is a subclass of ``int`` in Python, so a bare ``isinstance`` check reads
+    ``delay_seconds = true`` as the integer 1 and accepts it. Every integer-valued
+    configuration key goes through here so that trap is guarded in exactly one place.
+    """
+    return isinstance(value, int) and not isinstance(value, bool)
+
+
+def _kind(value: Any) -> str:
+    return type(value).__name__