vnnlib-test-solver 2.0.0__py3-none-any.whl

vnnlib_test_solver/rules.py

@@ -0,0 +1,119 @@
+"""Rule matching: which configured answer applies to a query file.
+
+The ``[[rules]]`` array is what makes this a test solver rather than a solver.
+An answer is chosen by looking the query file up in the configuration, never by
+examining anything the file says, which is the whole of the arrangement: one
+installed binary, three configuration files, three answers.
+
+**First match wins**, in the order the rules are written. The alternative - the
+most specific pattern wins - needs a definition of specificity for two arbitrary
+globs, and every such definition is a surprise waiting for whoever writes the
+second rule. Written order is a rule a reader can apply by eye.
+
+**No matching rule is an error, not a default.** Silence in a configuration file
+is not a claim, and a conformance baseline that invents an answer is worse than
+one that refuses to give it.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from .config import Config
+from .errors import ConfigError
+
+
+@dataclass(frozen=True)
+class Rule:
+    """One configured answer, built from a validated ``[[rules]]`` entry.
+
+    Every key the schema permits is read here rather than at the point it is
+    first used, so this record and the validated schema stay one list that can
+    be compared by eye.
+
+    ``model_element_types`` is what the configuration claims the supplied model
+    files expose, keyed by variable name. It is the only source this package
+    has for section 5.3.2's second error condition, since nothing here opens an
+    ONNX file.
+    """
+
+    match: str
+    result: str
+    assignments: Mapping[str, Sequence[float]]
+    model_element_types: Mapping[str, str]
+    delay_seconds: int
+    stderr: Optional[str]
+    exit_code: Optional[int]
+    raw_stdout: Optional[str]
+    crash: Optional[bool]
+    """``None`` when the rule does not mention it, which is not the same as
+    ``False``.
+
+    An explicit ``crash = false`` is a rule saying *do not* crash, and it has to
+    be able to override a ``[injection]`` table that says otherwise. Storing
+    both as ``False`` made those two indistinguishable and silently lost the
+    override - found in review, not by a test.
+    """
+
+
+def select(config: Config, query: Path) -> Rule:
+    """Return the first configured rule matching ``query``.
+
+    Raises:
+        ConfigError: no rule matches, so the configuration has no answer to give.
+    """
+    for entry in config.rules:
+        if matches(entry["match"], query):
+            return _rule(entry)
+    raise ConfigError(_no_match_message(config, query))
+
+
+def matches(pattern: str, query: Path) -> bool:
+    """Compare one pattern against the query file's name.
+
+    ``fnmatchcase`` rather than ``fnmatch``: the latter folds case through
+    ``os.path.normcase``, which does nothing on POSIX and lower-cases on
+    Windows. The same configuration and the same query would then answer
+    differently on different machines, and a conformance baseline that is
+    platform-dependent is not a baseline.
+
+    The file's name rather than the path it was given by, so a configuration
+    keeps working from whichever directory a caller happens to run in.
+    """
+    return fnmatch.fnmatchcase(query.name, pattern)
+
+
+def _no_match_message(config: Config, query: Path) -> str:
+    """Say what was matched, against what, so a near miss is diagnosable.
+
+    A pattern written against a path rather than a file name fails silently
+    otherwise, and looks like the rule being ignored.
+    """
+    if not config.rules:
+        return (
+            f"no rule matches query file {query.name!r}: {config.source.path} "
+            f"defines no rules at all, so it has no answer to give"
+        )
+    patterns = ", ".join(repr(entry["match"]) for entry in config.rules)
+    return (
+        f"no rule matches query file {query.name!r}; a pattern is compared against "
+        f"the file name alone, and {config.source.path} defines {patterns}"
+    )
+
+
+def _rule(entry: Mapping[str, Any]) -> Rule:
+    return Rule(
+        match=entry["match"],
+        result=entry["result"],
+        assignments=entry.get("assignments", {}),
+        model_element_types=entry.get("model_element_types", {}),
+        delay_seconds=entry.get("delay_seconds", 0),
+        stderr=entry.get("stderr"),
+        exit_code=entry.get("exit_code"),
+        raw_stdout=entry.get("raw_stdout"),
+        crash=entry.get("crash"),
+    )

vnnlib_test_solver/spec.py

@@ -0,0 +1,79 @@
+"""Validation targets transcribed from the standard.
+
+Theory identifier sets are section 4.1; element type names are the ONNX Set 1 names
+declarable in a query, plus the non-ONNX ``real``. Sets are frozen because nothing may
+extend them at runtime, and every message that names one sorts it first: unordered
+iteration in emitted output would break determinism.
+"""
+
+from __future__ import annotations
+
+HIDDEN_NODE_THEORIES = frozenset({"NH", "H"})
+MULTIPLE_INPUT_OUTPUT_THEORIES = frozenset({"SIO", "MIO"})
+MULTIPLE_NETWORK_THEORIES = frozenset({"SNET", "MNET", "MENET", "MINET"})
+MULTIPLE_NODE_COMPARISON_THEORIES = frozenset({"SNC", "MNC"})
+ARITHMETIC_COMPLEXITY_THEORIES = frozenset({"BND", "OUTC", "LIN", "POLY"})
+
+REAL = "real"
+"""The one element type the standard defines that ONNX does not.
+
+Named for the reason TIMED_OUT and SAT are: section 5.3.2 exempts a
+declaration written in this type from two of its three error conditions **by
+name**, so this package now acts on the value rather than only listing it
+among the others. A literal repeated in three places is free to drift away
+from the set it is supposed to belong to.
+
+Writing a declaration in this type is how section 3.5 says a user consents to
+analysis that may be unsound under floating point, which is why the exemption
+exists at all: there is nothing left to warn them about.
+"""
+
+ELEMENT_TYPES = frozenset(
+    {
+        REAL,
+        "float16",
+        "float32",
+        "float64",
+        "bfloat16",
+        "float8e4m3fn",
+        "float8e5m2",
+        "float8e4m3fnuz",
+        "float8e5m2fnuz",
+        "float4e2m1",
+        "int8",
+        "int16",
+        "int32",
+        "int64",
+        "uint8",
+        "uint16",
+        "uint32",
+        "uint64",
+        "complex64",
+        "complex128",
+        "bool",
+    }
+)
+
+TIMED_OUT = "timed-out"
+"""Section 5.3.1's answer for a solver that ran out of the time it was given.
+
+Named because it is the one result this package produces without being asked
+for it, when a rule's delay outlasts the caller's timeout. The other three are
+only ever quoted from configuration.
+"""
+
+SAT = "sat"
+"""Section 5.3.1's answer for a query the solver found satisfiable.
+
+Named for the same reason TIMED_OUT is: this package acts on it rather than
+only quoting it back, because section 5.3.1 attaches the assignment block to
+this result and to no other. A literal in two places is a literal that can
+drift apart from the set below.
+"""
+
+RESULTS = (TIMED_OUT, "unknown", "unsat", SAT)
+
+
+def as_list(values: frozenset[str]) -> str:
+    """Render a set for an error message in a stable order."""
+    return ", ".join(sorted(values))

vnnlib_test_solver/supports.py

@@ -0,0 +1,151 @@
+"""The supports command: capability responses, per standard section 5.4.
+
+Eleven mandatory capabilities in four distinct response shapes: a pair of version
+strings, a newline-separated list, one line per ONNX operator, and a boolean
+literal. Only two of the eleven are booleans, so each shape is rendered on its
+own terms rather than through one generic path.
+
+Every answer comes from configuration. A capability the configuration does not
+report is an error rather than a guess: a conformance baseline that invents an
+answer is worse than one that refuses to give it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any
+
+from .config import Config
+from .errors import ConfigError
+
+OPSET_VERSIONS = "--onnx-opset-versions"
+ELEMENT_TYPES = "--onnx-element-types"
+OPERATORS = "--onnx-operators"
+VNNLIB_VERSIONS = "--vnnlib-versions"
+HIDDEN_NODE = "--hidden-node-theories"
+MULTIPLE_INPUT_OUTPUT = "--multiple-input-output-theories"
+MULTIPLE_NETWORK = "--multiple-network-theories"
+MULTIPLE_NODE_COMPARISON = "--multiple-node-comparison-theories"
+ARITHMETIC_COMPLEXITY = "--arithmetic-complexity-theories"
+OPTIMISED_DISJUNCTIVE_REASONING = "--optimised-disjunctive-reasoning"
+SERIALISE_ASSIGNMENTS = "--serialise-assignments"
+
+CAPABILITIES = (
+    OPSET_VERSIONS,
+    ELEMENT_TYPES,
+    OPERATORS,
+    VNNLIB_VERSIONS,
+    HIDDEN_NODE,
+    MULTIPLE_INPUT_OUTPUT,
+    MULTIPLE_NETWORK,
+    MULTIPLE_NODE_COMPARISON,
+    ARITHMETIC_COMPLEXITY,
+    OPTIMISED_DISJUNCTIVE_REASONING,
+    SERIALISE_ASSIGNMENTS,
+)
+"""Every capability, in the order section 5.4 introduces them.
+
+A tuple rather than a set: this order reaches stderr in the message naming the
+legal capabilities, and unordered iteration in emitted output is not permitted.
+"""
+
+MULTIPLE_INPUT_OUTPUT_ALIAS = "--multiple-inputs-output-theories"
+"""Section 5.4.2 spells the flag one way in its heading and another, with an
+extra 's', in its own example. Both are accepted; the heading spelling is the
+documented one."""
+
+VERSION_PAIR_CAPABILITIES = (OPSET_VERSIONS, VNNLIB_VERSIONS)
+LIST_CAPABILITIES = (
+    ELEMENT_TYPES,
+    HIDDEN_NODE,
+    MULTIPLE_INPUT_OUTPUT,
+    MULTIPLE_NETWORK,
+    MULTIPLE_NODE_COMPARISON,
+    ARITHMETIC_COMPLEXITY,
+)
+BOOLEAN_CAPABILITIES = (OPTIMISED_DISJUNCTIVE_REASONING, SERIALISE_ASSIGNMENTS)
+
+
+def config_key(capability: str) -> str:
+    """The [capabilities] key that answers a capability flag.
+
+    Every flag in section 5.4 is its configuration key with the option dashes
+    removed, so the published flag and the documented key cannot drift apart.
+    """
+    return capability[2:]
+
+
+def response_shape(capability: str) -> str:
+    """Describe a capability's response shape, for the command's own help."""
+    if capability in VERSION_PAIR_CAPABILITIES:
+        return "two version strings, minimum then maximum"
+    if capability in BOOLEAN_CAPABILITIES:
+        return "true or false"
+    if capability == OPERATORS:
+        return "one operator per line, with the element types it supports"
+    if capability == ELEMENT_TYPES:
+        return "one element type per line"
+    return "one theory identifier per line"
+
+
+def render(capability: str, config: Config) -> str:
+    """Return the exact stdout text answering one capability.
+
+    The trailing newline is part of the answer. A capability configured as an
+    empty list renders as the empty string: supporting nothing is an answer, and
+    a blank line would not be one.
+    """
+    reported = _reported(capability, config)
+
+    if capability in LIST_CAPABILITIES:
+        return _lines(reported)
+
+    if capability in VERSION_PAIR_CAPABILITIES:
+        # Two lines, minimum then maximum. Configuration validation has already
+        # rejected any count other than two, so the shape is guaranteed here.
+        return _lines(reported)
+
+    if capability == OPERATORS:
+        return _operator_lines(reported)
+
+    if capability in BOOLEAN_CAPABILITIES:
+        # The specification's literals, not Python's: bool renders as True/False.
+        return "true\n" if reported else "false\n"
+
+    # Unreachable through the command line, where the flags are built from
+    # CAPABILITIES and a test asserts the four shapes above partition it. Kept so
+    # that adding a twelfth capability fails loudly rather than silently.
+    raise ConfigError(f"{capability} has no defined response shape")
+
+
+def _reported(capability: str, config: Config) -> Any:
+    key = config_key(capability)
+    if key not in config.capabilities:
+        raise ConfigError(
+            f"the configuration does not report {capability}; add {key!r} under "
+            f"[capabilities] in {config.source.path}"
+        )
+    return config.capabilities[key]
+
+
+def _operator_lines(operators: Sequence[Mapping[str, Any]]) -> str:
+    """One line per operator: its name, then the element types it is supported for.
+
+    An operator configured with no element types is emitted bare. Section 5.4.1
+    states that where the list is empty, the solver is assumed to support the
+    operator for all element types it reports through --onnx-element-types, so
+    the bare line is the standard's own way of saying "all of them", not "none".
+
+    That assumption is the reader's to make, which is why the line is left bare
+    rather than expanded: expanding it would answer the same question with
+    different bytes and settle on a consumer's behalf something the standard
+    leaves to their parser.
+    """
+    return _lines(
+        " ".join([operator["name"], *operator.get("element-types", [])])
+        for operator in operators
+    )
+
+
+def _lines(values: Iterable[str]) -> str:
+    return "".join(f"{value}\n" for value in values)