vnnlib-test-solver 2.0.0__py3-none-any.whl

vnnlib_test_solver/config.py

@@ -0,0 +1,130 @@
+"""Configuration discovery and loading.
+
+Discovery order is the published contract: an explicit ``--config`` path, then the
+environment variable, then a file in the working directory.
+
+**There is no packaged fallback, and running with no configuration at all is an
+error.** Every answer this program gives - its name, its version, each capability
+and each result - is a configured answer, so with no configuration there is
+nothing it can honestly say. Shipping a default would mean answering anyway, and
+an answer nobody configured is exactly what a conformance baseline must never
+produce: it would be the one reply in the system that no test author chose, and
+it would look identical to one they did.
+
+A source named explicitly and then found missing is an error rather than a
+fall-through to the next source. Falling through would let a mistyped path
+silently answer from somewhere else, which is the same failure in a smaller
+place.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from .errors import ConfigError
+from .validation import validate
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+ENV_VAR = "VNNLIB_TEST_SOLVER_CONFIG"
+WORKING_DIRECTORY_NAME = "vnnlibTestSolver.toml"
+
+
+@dataclass(frozen=True)
+class ConfigSource:
+    """Where a configuration was resolved from, and how to describe that on stderr."""
+
+    path: Path
+    origin: str
+
+
+@dataclass(frozen=True)
+class Solver:
+    """The ``[solver]`` table: the answers to the two global options."""
+
+    name: str
+    version: str
+
+
+@dataclass(frozen=True)
+class Config:
+    """A validated configuration.
+
+    Sections consumed by later commands stay as mappings: their typed shape belongs
+    with the code that reads them, not here.
+    """
+
+    source: ConfigSource
+    solver: Solver
+    capabilities: Mapping[str, Any]
+    soundness: Mapping[str, Any]
+    injection: Mapping[str, Any]
+    rules: tuple[Mapping[str, Any], ...]
+
+
+def resolve_config_source(explicit: Optional[str] = None) -> ConfigSource:
+    """Resolve the configuration file to use, in the documented precedence order.
+
+    Raises:
+        ConfigError: no source applies, so there is no configured answer to give.
+    """
+    if explicit is not None:
+        return _named_source(explicit, "--config")
+
+    from_env = os.environ.get(ENV_VAR)
+    if from_env:
+        return _named_source(from_env, f"${ENV_VAR}")
+
+    in_working_directory = Path.cwd() / WORKING_DIRECTORY_NAME
+    if in_working_directory.is_file():
+        return ConfigSource(in_working_directory, "working directory")
+
+    # The message names all three sources rather than only the last one tried,
+    # because a caller who has set one of them and still landed here has set the
+    # wrong one, and cannot tell which from a message about the working
+    # directory alone.
+    raise ConfigError(
+        "no configuration file found, and this solver answers only from "
+        f"configuration: give --config <path>, set ${ENV_VAR}, or put a "
+        f"{WORKING_DIRECTORY_NAME} in the working directory"
+    )
+
+
+def _named_source(candidate: str, origin: str) -> ConfigSource:
+    path = Path(candidate)
+    if not path.is_file():
+        raise ConfigError(f"configuration file given by {origin} does not exist: {path}")
+    return ConfigSource(path, origin)
+
+
+def load_config(source: ConfigSource) -> Config:
+    """Read, parse and validate the configuration at ``source``."""
+    try:
+        with source.path.open("rb") as handle:
+            raw = tomllib.load(handle)
+    except OSError as exc:
+        raise ConfigError(f"cannot read configuration file {source.path}: {exc}") from exc
+    except tomllib.TOMLDecodeError as exc:
+        raise ConfigError(
+            f"configuration file {source.path} is not valid TOML: {exc}"
+        ) from exc
+
+    validate(raw)
+
+    solver = raw["solver"]
+    return Config(
+        source=source,
+        solver=Solver(name=solver["name"], version=solver["version"]),
+        capabilities=raw.get("capabilities", {}),
+        soundness=raw.get("soundness", {}),
+        injection=raw.get("injection", {}),
+        rules=tuple(raw.get("rules", [])),
+    )

vnnlib_test_solver/dtypes.py

@@ -0,0 +1,67 @@
+"""The declarable element types, keyed by the parser's own enum member name.
+
+Section 5.3.1 prints each variable's type in the specification's string form,
+``float32``, not in the form the parser's enum reprs it as. No reverse map
+exists in either library - ``VNNLIB-CPP``'s type checker maps string to enum
+and never back - so this one is maintained by hand and checked against the
+enum by a test rather than trusted.
+
+**Keyed by the member's name rather than by the member itself**, so this module
+holds no ``vnnlib`` import. Only ``querymodel`` may have one, and a table the
+rest of the package can read without touching the parser is what lets the
+parser stay behind that one file.
+
+Twenty-one entries: the twenty ONNX Set 1 names declarable in a query, plus the
+non-ONNX ``real``. The enum carries five further members - constants and the
+unknown placeholder - which no declaration can produce, so a lookup that misses
+is a type the standard does not define, whatever caused it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+DTYPE_TO_SPEC: Mapping[str, str] = {
+    "Real": "real",
+    "F16": "float16",
+    "F32": "float32",
+    "F64": "float64",
+    "BF16": "bfloat16",
+    "F8E4M3FN": "float8e4m3fn",
+    "F8E5M2": "float8e5m2",
+    "F8E4M3FNUZ": "float8e4m3fnuz",
+    "F8E5M2FNUZ": "float8e5m2fnuz",
+    "F4E2M1": "float4e2m1",
+    "I8": "int8",
+    "I16": "int16",
+    "I32": "int32",
+    "I64": "int64",
+    "U8": "uint8",
+    "U16": "uint16",
+    "U32": "uint32",
+    "U64": "uint64",
+    "C64": "complex64",
+    "C128": "complex128",
+    "Bool": "bool",
+}
+
+NOT_DECLARABLE = frozenset(
+    {
+        "FloatConstant",
+        "NegativeIntConstant",
+        "PositiveIntConstant",
+        "String",
+        "Unknown",
+    }
+)
+"""Enum members that exist but that no ``declare-`` form can produce.
+
+Four of them classify literals inside assertions. The fifth, ``Unknown``, is
+what the pinned parser returns for an element type name it does not recognise,
+silently and without raising, which is why a lookup miss has to be refused
+where the declaration is read.
+
+Recorded here rather than left implicit so that the two halves stay one list:
+a test asserts that these five and the twenty-one above account for every
+member of the enum, so a member added upstream cannot slip through as either.
+"""

vnnlib_test_solver/errors.py

@@ -0,0 +1,30 @@
+"""Shared exception types.
+
+They are kept apart because they are different mistakes made by different
+people: a usage error is the caller's, a configuration error is the
+configuration author's, and a query error is the query author's. Distinct
+types are what let each carry its own exit code and its own wording.
+"""
+
+
+class ConfigError(Exception):
+    """A configuration file could not be resolved, parsed, or validated."""
+
+
+class UsageError(Exception):
+    """Command-line arguments parsed but do not describe a usable request.
+
+    For rejections this package makes itself. A rejection argparse makes leaves
+    through its own error handling and never becomes one of these.
+    """
+
+
+class QueryError(Exception):
+    """A query file could not be read, parsed, or understood.
+
+    Neither of the other two: the command line named a file and the
+    configuration said what to answer, and the fault is in the query itself.
+    Kept separate so that a caller debugging a fixture is not told to look at
+    its configuration, and so the exit code this maps to can be decided once,
+    where the command handles it.
+    """

vnnlib_test_solver/injection.py

@@ -0,0 +1,119 @@
+"""Deliberate misbehaviour, resolved from configuration.
+
+A consumer's handling of a solver that returns unparseable output, exits with a
+code that disagrees with its answer, writes noise on ``stderr``, or dies rather
+than answering is the code least likely to be exercised and most likely to be
+wrong. No correct solver produces any of it on request, so a test solver that
+cannot be told to is missing the half of the interface that is hardest to test.
+
+**Two places, one set of keys.** A ``[[rules]]`` entry carries them, because
+which answer a query gets is already chosen per rule and misbehaviour belongs
+with the answer it accompanies. The top-level ``[injection]`` table carries the
+same four, because a rule is selected by matching a query file's name and the
+``supports`` command has no query file - so nothing hanging off a rule can ever
+reach it, and the capability responses are exactly what a consumer's other
+parser needs to be fed malformed.
+
+Where both set the same control the rule wins, **per control rather than
+wholesale**. Replacing the whole table would mean a rule that only wanted a
+different exit code silently discarded the ``stderr`` noise configured beside
+it, which is the kind of quiet subtraction that makes a fixture describe one
+thing and test another.
+
+Nothing here writes to a stream. Resolution is separated from delivery so that
+what a configuration means can be tested without capturing output, and so that
+the one module permitted to write to ``stdout`` stays the one that does.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, Optional
+
+KEYS = frozenset({"stderr", "exit_code", "raw_stdout", "crash"})
+"""The four controls, named once.
+
+``validation`` builds both the rule schema and the ``[injection]`` schema from
+this set rather than listing the names twice, so the two cannot drift apart and
+then disagree about which keys exist.
+"""
+
+
+@dataclass(frozen=True)
+class Injection:
+    """What a run should do instead of behaving correctly.
+
+    ``None`` means "not asked for" rather than "asked for nothing": an empty
+    ``raw_stdout`` is a real instruction to print nothing at all, and is not the
+    same as leaving the key out.
+    """
+
+    stderr: Optional[str] = None
+    exit_code: Optional[int] = None
+    raw_stdout: Optional[str] = None
+    crash: bool = False
+
+
+NOTHING = Injection()
+"""The absence of any instruction, for a run nothing was configured for."""
+
+
+def resolve(table: Mapping[str, Any], rule: Optional[Any] = None) -> Injection:
+    """Combine the ``[injection]`` table with a rule's own controls.
+
+    ``rule`` is anything carrying the four attributes - a ``Rule`` in practice.
+    It is typed loosely on purpose: this module is imported by ``rules`` in
+    neither direction, and giving it a concrete dependency on that record would
+    buy nothing and cost a cycle.
+    """
+    crash = _pick(table, rule, "crash")
+    exit_code = _pick(table, rule, "exit_code")
+
+    # A process that terminates abnormally reports no exit status, so these two
+    # cannot both happen. Within one table that is a configuration error, caught
+    # when the file loads. Across the two levels it is not: each half is valid
+    # where it is written, and neither validator can see the other. It is
+    # resolved here instead, by the rule that governs every other control -
+    # the more specific source wins - so that nothing is silently ignored at
+    # the point of delivery, where nobody would see it happen.
+    #
+    # Only a crash actually being asked for conflicts with an exit code. An
+    # explicit ``crash = false`` conflicts with nothing: it is a rule declining
+    # to crash, which is precisely what lets it override a table that does.
+    if crash and exit_code is not None:
+        if _stated_by_rule(rule, "crash"):
+            exit_code = None
+        else:
+            crash = False
+
+    return Injection(
+        stderr=_pick(table, rule, "stderr"),
+        exit_code=exit_code,
+        raw_stdout=_pick(table, rule, "raw_stdout"),
+        crash=bool(crash),
+    )
+
+
+def _stated_by_rule(rule: Optional[Any], key: str) -> bool:
+    """Whether the rule mentioned this control at all, as opposed to leaving it out."""
+    return rule is not None and getattr(rule, key, None) is not None
+
+
+def _pick(table: Mapping[str, Any], rule: Optional[Any], key: str) -> Any:
+    """The rule's value for one control if it stated one, else the table's.
+
+    Every control uses ``None`` for "not stated", which is what makes this one
+    comparison correct for all four. ``crash`` used to be the exception - its
+    absent form was ``False``, so this function compared against each key's own
+    absent value instead. That protected a rule which simply omits ``crash``
+    from overriding a table that sets it, but it also made an explicit
+    ``crash = false`` indistinguishable from silence, so a rule could not turn
+    a table's crash off. Found in review; the fix was to give ``crash`` the same
+    absent value as everything else rather than to special-case it here.
+    """
+    if rule is not None:
+        value = getattr(rule, key, None)
+        if value is not None:
+            return value
+    return table.get(key)

vnnlib_test_solver/querymodel.py

@@ -0,0 +1,250 @@
+"""Reading a query file: the only module permitted to import ``vnnlib``.
+
+Section 5.3.1 requires a satisfying assignment to name each variable, its type
+and its dimensions, in the order the query file declares them. None of that is
+in the configuration file and none of it may be: a configuration whose variable
+metadata silently disagreed with its query would produce a confidently wrong
+answer, which is the one failure a conformance baseline must never have.
+Configuration supplies values; the query supplies structure.
+
+Everything the rest of the package sees from here is a plain string, integer or
+tuple. No parser object, and no name from the parser's vocabulary, crosses this
+boundary. That is what makes the dependency reversible: were the package
+required to be standalone, this file would read structure from configuration
+instead and every other module would keep its interface.
+
+Declaration order is the walk ``inputs``, then hidden nodes, then ``outputs``,
+per network in the order the networks are declared. That is not a convention
+chosen here. The grammar orders those three groups positionally, so any other
+order is a parse error rather than a differently ordered parse, and the walk
+reproduces the standard's own worked example exactly.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from .dtypes import DTYPE_TO_SPEC
+from .errors import QueryError
+from .spec import ELEMENT_TYPES, as_list
+
+try:
+    # Where the public API is moving. Every name still reachable at the top
+    # level emits a DeprecationWarning once that move lands, including through
+    # a plain from-import, because a from-import falls back to the module's
+    # __getattr__. Trying the new location first means this file is correct on
+    # both sides of that migration and warns on neither.
+    from vnnlib.query import VNNLibException, parse_query_file
+except ImportError:  # pragma: no cover - depends on the installed version
+    # The pinned version, which has no vnnlib.query at all.
+    from vnnlib import VNNLibException, parse_query_file
+
+INPUT = "input"
+HIDDEN = "hidden"
+OUTPUT = "output"
+
+
+@dataclass(frozen=True)
+class Declaration:
+    """One declared variable, as the assignment output needs to describe it.
+
+    ``element_type`` is the specification's string form rather than the
+    parser's enum, so that a reader of this record never has to know the
+    parser exists. ``shape`` is a tuple rather than the list the parser hands
+    back, because that list is the caller's to mutate and this record is not.
+    """
+
+    name: str
+    element_type: str
+    shape: tuple[int, ...]
+    kind: str
+    network: str
+
+
+@dataclass(frozen=True)
+class Network:
+    """One declared network, and what it is declared equivalent to.
+
+    ``equal_to`` decides whether a network needs an ONNX model supplied for
+    it; the parser reports an absent equivalence as an empty string, which is
+    narrowed to ``None`` here so that absence cannot be mistaken for a network
+    named by the empty string.
+
+    ``isometric_to`` is carried alongside it and deliberately does **not**
+    exempt a network from the mapping. An isomorphic pair shares a graph
+    structure and has different weights - the standard's motivating cases are
+    retraining and quantisation - so each one needs its own model file, and
+    section 5.3 exempts only ``equal-to`` by name. Carrying the field is what
+    forced that to be answered against the standard rather than discovered
+    missing later. The standard spells this *isomorphic*; the parser's
+    attribute spells it *isometric*, and they are the same concept.
+    """
+
+    name: str
+    equal_to: Optional[str]
+    isometric_to: Optional[str]
+
+
+@dataclass(frozen=True)
+class QueryModel:
+    """A query file, reduced to what answering about it requires."""
+
+    path: Path
+    networks: tuple[Network, ...]
+    declarations: tuple[Declaration, ...]
+
+
+def read_query(path: Path) -> QueryModel:
+    """Read ``path`` and return its networks and declarations, in order.
+
+    Raises:
+        QueryError: the file cannot be read, does not parse, fails the
+            parser's own type checking, or declares an element type the
+            standard does not define.
+    """
+    _require_readable(path)
+
+    try:
+        parsed = parse_query_file(str(path))
+    except VNNLibException as exc:
+        raise QueryError(f"{path} is not a valid query: {_explain(exc)}") from exc
+
+    # Not every failure raises. Given a path it cannot open, the parser writes
+    # its own message and returns None, so a caller that goes straight for an
+    # attribute gets a traceback rather than a message. The readable check above
+    # is what normally prevents this; the guard covers whatever it does not,
+    # such as a file that becomes unreadable between the two.
+    #
+    # The published stub declares this function as returning a query and never
+    # None, so nothing warns about the branch and a type checker will not put it
+    # there for you. Verified by running the parser against a path that does not
+    # exist: it returns None.
+    if parsed is None:
+        raise QueryError(
+            f"the query parser could not read {path} and gave no reason; "
+            f"check that the file is readable and is a VNN-LIB query"
+        )
+
+    networks = tuple(_network(network) for network in parsed.networks)
+    declarations = tuple(_declarations(parsed, path))
+    return QueryModel(path=path, networks=networks, declarations=declarations)
+
+
+def _require_readable(path: Path) -> None:
+    """Refuse an unreadable path here, where it can still be explained.
+
+    The parser does not raise for one. It prints a message of its own and
+    returns None, so refusing first is what keeps every message this package
+    produces in this package's wording, and keeps the parser's C++ message off
+    a stream a consumer is reading.
+    """
+    if not path.exists():
+        raise QueryError(f"query file does not exist: {path}")
+    if not path.is_file():
+        raise QueryError(f"query file is not a file: {path}")
+
+
+def _network(network: Any) -> Network:
+    return Network(
+        name=network.name,
+        equal_to=network.equal_to or None,
+        isometric_to=network.isometric_to or None,
+    )
+
+
+def _declarations(parsed: Any, path: Path) -> list[Declaration]:
+    """Walk every network's declarations in the order the grammar fixes."""
+    declarations: list[Declaration] = []
+    for network in parsed.networks:
+        groups = (
+            (INPUT, network.inputs),
+            (HIDDEN, network.hidden),
+            (OUTPUT, network.outputs),
+        )
+        for kind, definitions in groups:
+            for definition in definitions:
+                declarations.append(_declaration(definition, kind, network.name, path))
+    return declarations
+
+
+def _declaration(definition: Any, kind: str, network: str, path: Path) -> Declaration:
+    element_type = DTYPE_TO_SPEC.get(definition.dtype.name)
+    if element_type is None:
+        # The parser accepts an element type name it does not recognise without
+        # raising, and reports it as its unknown placeholder. Left alone that
+        # produces a header line that is wrong and entirely plausible, so it is
+        # refused before anything can be printed. The name as written is not
+        # recoverable from the parsed form, which is why the message names the
+        # variable and the legal set instead of the offending word.
+        raise QueryError(
+            f"{path}: {kind} {definition.name!r} in network {network!r} is declared "
+            f"with an element type the standard does not define; element type names "
+            f"are lower case and case sensitive, and the standard defines "
+            f"{as_list(ELEMENT_TYPES)}"
+        )
+    return Declaration(
+        name=definition.name,
+        element_type=element_type,
+        shape=_shape(definition.shape),
+        kind=kind,
+        network=network,
+    )
+
+
+def _shape(dimensions: Sequence[int]) -> tuple[int, ...]:
+    return tuple(int(dimension) for dimension in dimensions)
+
+
+def _explain(exc: VNNLibException) -> str:
+    """Render one parser exception, whichever of its two body formats it used.
+
+    A parse error carries plain text and a semantic error carries a JSON
+    report, both from this one exception type. Reading either as the other
+    raises from inside the error handler, so which one it is has to be
+    established before the body is read rather than assumed.
+
+    Errors are reported in the order the parser gives them, never sorted: the
+    same query must produce the same message every time it is refused.
+    """
+    body = str(exc)
+    try:
+        report = json.loads(body)
+    except ValueError:
+        return body
+
+    if not isinstance(report, dict):
+        return body
+    errors = report.get("errors")
+    if not isinstance(errors, list) or not errors:
+        # A report with nothing in it should not happen: the parser raises this
+        # format only when it has errors to describe. Falling back to the body
+        # would print raw JSON at whoever is reading stderr, which is the one
+        # thing this function exists to avoid, so it says what is known instead.
+        return (
+            "the query failed the parser's checks, but the report carried no "
+            "detail about why"
+        )
+
+    first = _describe(errors[0])
+    if len(errors) == 1:
+        return first
+    return f"{first} (and {len(errors) - 1} further problem(s) in the same file)"
+
+
+def _describe(error: Any) -> str:
+    """Turn one entry of a semantic error report into a line of prose."""
+    if not isinstance(error, dict):
+        return str(error)
+    parts = []
+    line = error.get("line")
+    if line is not None:
+        parts.append(f"line {line}")
+    for key in ("message", "hint"):
+        value = error.get(key)
+        if value:
+            parts.append(str(value))
+    return ": ".join(parts) if parts else str(error)