vnnlib-test-solver 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vnnlib_test_solver/__init__.py +1 -0
- vnnlib_test_solver/assignments.py +163 -0
- vnnlib_test_solver/cli.py +504 -0
- vnnlib_test_solver/config.py +130 -0
- vnnlib_test_solver/dtypes.py +67 -0
- vnnlib_test_solver/errors.py +30 -0
- vnnlib_test_solver/injection.py +119 -0
- vnnlib_test_solver/py.typed +0 -0
- vnnlib_test_solver/querymodel.py +250 -0
- vnnlib_test_solver/rules.py +119 -0
- vnnlib_test_solver/spec.py +79 -0
- vnnlib_test_solver/supports.py +151 -0
- vnnlib_test_solver/validation.py +495 -0
- vnnlib_test_solver/verify.py +417 -0
- vnnlib_test_solver-2.0.0.dist-info/METADATA +176 -0
- vnnlib_test_solver-2.0.0.dist-info/RECORD +19 -0
- vnnlib_test_solver-2.0.0.dist-info/WHEEL +4 -0
- vnnlib_test_solver-2.0.0.dist-info/entry_points.txt +2 -0
- vnnlib_test_solver-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
"""The verify command, per standard section 5.3.
|
|
2
|
+
|
|
3
|
+
Section 5.3 fixes the shape of the command::
|
|
4
|
+
|
|
5
|
+
<solver> verify <filepath> [--network <name>=<filepath>]
|
|
6
|
+
[--timeout <seconds>]
|
|
7
|
+
[--serialise-assignments <filepath>]
|
|
8
|
+
|
|
9
|
+
Nothing here opens the query file. A request is a record of what was asked
|
|
10
|
+
for, and which rule answers it is decided by the file's name alone. The one
|
|
11
|
+
function that needs the query's contents is handed a model that has already
|
|
12
|
+
been read, so the parser stays behind the single module that imports it.
|
|
13
|
+
|
|
14
|
+
Section 5.3 permits a solver to add its own arguments so long as they are
|
|
15
|
+
optional. This one adds none: on a conformance baseline, every spelling that is
|
|
16
|
+
not in the standard is surface a consumer can come to depend on.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import argparse
|
|
22
|
+
import time
|
|
23
|
+
from collections.abc import Mapping, Sequence
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Optional
|
|
27
|
+
|
|
28
|
+
from .errors import ConfigError, UsageError
|
|
29
|
+
from .querymodel import QueryModel
|
|
30
|
+
from .rules import Rule
|
|
31
|
+
from .spec import REAL, TIMED_OUT
|
|
32
|
+
from .validation import MAX_WAIT_SECONDS
|
|
33
|
+
|
|
34
|
+
VERIFY_COMMAND = "verify"
|
|
35
|
+
|
|
36
|
+
QUERY_ARGUMENT = "<filepath>"
|
|
37
|
+
NETWORK_OPTION = "--network"
|
|
38
|
+
TIMEOUT_OPTION = "--timeout"
|
|
39
|
+
SERIALISE_ASSIGNMENTS_OPTION = "--serialise-assignments"
|
|
40
|
+
|
|
41
|
+
SERIALISE_ASSIGNMENTS_ALIAS = "--assignments"
|
|
42
|
+
"""Section 5.3's usage block spells the option this way, while its own option
|
|
43
|
+
description and section 5.4.3 both spell it --serialise-assignments. Which one a
|
|
44
|
+
conformant solver must accept is an open question with the client, so both are
|
|
45
|
+
accepted and the longer spelling is the documented one; that way neither reading
|
|
46
|
+
of the standard breaks a consumer. *Assumption*, pending an answer."""
|
|
47
|
+
|
|
48
|
+
VERIFY_OPTIONS = (NETWORK_OPTION, TIMEOUT_OPTION, SERIALISE_ASSIGNMENTS_OPTION)
|
|
49
|
+
"""The options section 5.3 defines, in the order it lists them.
|
|
50
|
+
|
|
51
|
+
A tuple rather than a set: this order reaches stderr in the message naming the
|
|
52
|
+
legal options, and unordered iteration in emitted output is not permitted.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class VerifyRequest:
|
|
58
|
+
"""One well-formed invocation of the verify command.
|
|
59
|
+
|
|
60
|
+
``networks`` keeps command-line order rather than sorting: it is emitted in
|
|
61
|
+
error messages about the mapping, and a sort would make the wording depend on
|
|
62
|
+
the names rather than on what the caller wrote.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
query: Path
|
|
66
|
+
networks: Mapping[str, Path]
|
|
67
|
+
timeout: Optional[int]
|
|
68
|
+
serialise_assignments: Optional[Path]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def add_arguments(command: argparse.ArgumentParser) -> None:
|
|
72
|
+
"""Declare the section 5.3 arguments on the verify sub-parser."""
|
|
73
|
+
command.add_argument(
|
|
74
|
+
"query",
|
|
75
|
+
metavar=QUERY_ARGUMENT,
|
|
76
|
+
help="path to the VNN-LIB query file to verify",
|
|
77
|
+
)
|
|
78
|
+
command.add_argument(
|
|
79
|
+
NETWORK_OPTION,
|
|
80
|
+
metavar="<name>=<filepath>",
|
|
81
|
+
action="append",
|
|
82
|
+
dest="networks",
|
|
83
|
+
default=[],
|
|
84
|
+
help=(
|
|
85
|
+
"map a declared network to an ONNX model file; given once for each "
|
|
86
|
+
"network declared without an equal-to"
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
# type=int makes argparse itself reject a non-integer, which exits through the
|
|
90
|
+
# parser's own error() and so already carries this package's usage exit code.
|
|
91
|
+
# It does not reject a negative, which parses cleanly and means nothing, so the
|
|
92
|
+
# range is checked separately and refused in this package's own wording.
|
|
93
|
+
command.add_argument(
|
|
94
|
+
TIMEOUT_OPTION,
|
|
95
|
+
metavar="<seconds>",
|
|
96
|
+
type=int,
|
|
97
|
+
help="whole seconds to allow before the solver answers timed-out",
|
|
98
|
+
)
|
|
99
|
+
command.add_argument(
|
|
100
|
+
SERIALISE_ASSIGNMENTS_OPTION,
|
|
101
|
+
metavar="<filepath>",
|
|
102
|
+
dest="serialise_assignments",
|
|
103
|
+
help="folder to write serialised assignments into, instead of stdout",
|
|
104
|
+
)
|
|
105
|
+
command.add_argument(
|
|
106
|
+
SERIALISE_ASSIGNMENTS_ALIAS,
|
|
107
|
+
metavar="<filepath>",
|
|
108
|
+
dest="serialise_assignments",
|
|
109
|
+
help=argparse.SUPPRESS,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def request_from(args: argparse.Namespace) -> VerifyRequest:
|
|
114
|
+
"""Build a request from parsed arguments, refusing what argparse cannot.
|
|
115
|
+
|
|
116
|
+
Raises:
|
|
117
|
+
UsageError: the arguments parsed but do not describe a usable request.
|
|
118
|
+
"""
|
|
119
|
+
# argparse accepts an empty string for a positional, so this is the only place
|
|
120
|
+
# it can be refused. A file with no name cannot exist, and matching it against
|
|
121
|
+
# a catch-all pattern would otherwise produce a confident answer about nothing.
|
|
122
|
+
if not args.query:
|
|
123
|
+
raise UsageError(
|
|
124
|
+
f"the {QUERY_ARGUMENT} argument must not be empty; {VERIFY_COMMAND} "
|
|
125
|
+
f"needs the path to a query file"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
timeout = args.timeout
|
|
129
|
+
if timeout is not None and timeout < 0:
|
|
130
|
+
raise UsageError(
|
|
131
|
+
f"{TIMEOUT_OPTION} must be zero or more whole seconds, not {timeout}"
|
|
132
|
+
)
|
|
133
|
+
# Above this the wait itself raises, and an unhandled exception reaches the
|
|
134
|
+
# caller as a traceback rather than as a message. See MAX_WAIT_SECONDS.
|
|
135
|
+
if timeout is not None and timeout > MAX_WAIT_SECONDS:
|
|
136
|
+
raise UsageError(
|
|
137
|
+
f"{TIMEOUT_OPTION} must be at most {MAX_WAIT_SECONDS} seconds; "
|
|
138
|
+
f"{timeout} is longer than this package can wait"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
networks: dict[str, Path] = {}
|
|
142
|
+
for token in args.networks:
|
|
143
|
+
name, path = _network_pair(token)
|
|
144
|
+
if name in networks:
|
|
145
|
+
raise UsageError(
|
|
146
|
+
f"{NETWORK_OPTION} maps {name!r} more than once; each declared "
|
|
147
|
+
f"network takes exactly one model file"
|
|
148
|
+
)
|
|
149
|
+
networks[name] = path
|
|
150
|
+
|
|
151
|
+
serialise_assignments = args.serialise_assignments
|
|
152
|
+
return VerifyRequest(
|
|
153
|
+
query=Path(args.query),
|
|
154
|
+
networks=networks,
|
|
155
|
+
timeout=timeout,
|
|
156
|
+
serialise_assignments=(
|
|
157
|
+
None if serialise_assignments is None else Path(serialise_assignments)
|
|
158
|
+
),
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def warnings(request: VerifyRequest) -> tuple[str, ...]:
|
|
163
|
+
"""Report anything asked for that this build will not actually do.
|
|
164
|
+
|
|
165
|
+
Serialising assignments is optional in the standard and is not implemented
|
|
166
|
+
yet, but the option is accepted so that the published argument surface is the
|
|
167
|
+
one section 5.3 defines. Accepting it and silently doing nothing is the
|
|
168
|
+
dangerous half of that: a caller testing serialisation would see a zero exit,
|
|
169
|
+
no files, and no explanation, which reads as a pass.
|
|
170
|
+
|
|
171
|
+
A warning rather than an error, because the run still delivers a correct
|
|
172
|
+
result line and the standard makes command-line output the default that every
|
|
173
|
+
solver must support regardless of serialisation. Returned rather than printed,
|
|
174
|
+
the way the configuration warnings are, so this stays testable without
|
|
175
|
+
capturing a stream.
|
|
176
|
+
"""
|
|
177
|
+
if request.serialise_assignments is None:
|
|
178
|
+
return ()
|
|
179
|
+
return (
|
|
180
|
+
f"{SERIALISE_ASSIGNMENTS_OPTION} was given, but writing assignment files "
|
|
181
|
+
f"is not implemented yet, so nothing was written to "
|
|
182
|
+
f"{request.serialise_assignments}; the result is still on stdout",
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def answer(request: VerifyRequest, rule: Rule) -> str:
|
|
187
|
+
"""Wait as long as the rule asks for, then return the result to print.
|
|
188
|
+
|
|
189
|
+
A rule's delay is how long the solver claims to spend thinking, and the
|
|
190
|
+
caller's timeout is how long it is willing to wait. When the delay outlasts
|
|
191
|
+
the timeout the answer is ``timed-out``, and the wait is the timeout rather
|
|
192
|
+
than the delay: the run then finishes on the caller's deadline, which is the
|
|
193
|
+
only way a caller enforcing its own can be tested against this binary at all.
|
|
194
|
+
|
|
195
|
+
Waiting the whole timeout rather than returning at once is the point. A
|
|
196
|
+
solver that answers ``timed-out`` instantly is not one anybody's timeout
|
|
197
|
+
handling can be exercised against.
|
|
198
|
+
|
|
199
|
+
A delay exactly equal to the timeout answers rather than times out. The
|
|
200
|
+
timeout is where waiting has gone on too long, and waiting exactly that long
|
|
201
|
+
has not gone on too long yet.
|
|
202
|
+
"""
|
|
203
|
+
if request.timeout is not None and rule.delay_seconds > request.timeout:
|
|
204
|
+
time.sleep(request.timeout)
|
|
205
|
+
return TIMED_OUT
|
|
206
|
+
|
|
207
|
+
time.sleep(rule.delay_seconds)
|
|
208
|
+
return rule.result
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def is_real_valued(model: QueryModel) -> bool:
|
|
212
|
+
"""Whether the query is real-valued: every declaration written ``real``.
|
|
213
|
+
|
|
214
|
+
Section 3.5 defines a real-valued query as one that *"must exclusively use
|
|
215
|
+
the special real element type"*, and section 5.3.2 exempts such a query
|
|
216
|
+
from two of its three error conditions. Both conditions ask the same
|
|
217
|
+
question, so they ask it in one place: a single definition cannot come to
|
|
218
|
+
disagree with itself about what "the query uses real" means.
|
|
219
|
+
|
|
220
|
+
A query declaring nothing would answer ``True`` here on an empty ``all``.
|
|
221
|
+
The grammar makes that unreachable - every network declares at least one
|
|
222
|
+
input and one output - so it is left as the vacuous truth rather than
|
|
223
|
+
guarded against, and named here so the next reader does not have to work
|
|
224
|
+
out whether it was overlooked.
|
|
225
|
+
"""
|
|
226
|
+
return all(declaration.element_type == REAL for declaration in model.declarations)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def check_network_mapping(request: VerifyRequest, model: QueryModel) -> None:
|
|
230
|
+
"""Enforce section 5.3.2's first error condition on the supplied mapping.
|
|
231
|
+
|
|
232
|
+
One model file per declared network, except a network declared equivalent
|
|
233
|
+
to another, which is verified by verifying the network it copies and so
|
|
234
|
+
needs no model of its own. Its variables still appear in the assignment;
|
|
235
|
+
only this mapping exempts it.
|
|
236
|
+
|
|
237
|
+
The two directions are reported separately, and a name supplied for an
|
|
238
|
+
equivalent network is separated from a name the query never declares,
|
|
239
|
+
because a caller who gets one of those wrong has made a different mistake
|
|
240
|
+
from a caller who gets the other wrong. A single "wrong number of models"
|
|
241
|
+
message would also let a test pass against a solver that simply refused
|
|
242
|
+
everything.
|
|
243
|
+
|
|
244
|
+
Both lists are walked in a fixed order - declarations in the order the
|
|
245
|
+
query makes them, supplied names in the order the command line gave them -
|
|
246
|
+
so the message names the same network every time for the same invocation.
|
|
247
|
+
|
|
248
|
+
Raises:
|
|
249
|
+
UsageError: the supplied model files do not match the declarations.
|
|
250
|
+
"""
|
|
251
|
+
equivalences = {network.name: network.equal_to for network in model.networks}
|
|
252
|
+
required = [name for name, equal_to in equivalences.items() if equal_to is None]
|
|
253
|
+
|
|
254
|
+
for name in request.networks:
|
|
255
|
+
if name not in equivalences:
|
|
256
|
+
raise UsageError(
|
|
257
|
+
f"{NETWORK_OPTION} maps {name!r}, which {model.path} does not "
|
|
258
|
+
f"declare; a model file is supplied for each declared network"
|
|
259
|
+
)
|
|
260
|
+
if equivalences[name] is not None:
|
|
261
|
+
raise UsageError(
|
|
262
|
+
f"{NETWORK_OPTION} maps {name!r}, which {model.path} declares equal "
|
|
263
|
+
f"to {equivalences[name]!r}; an equivalent network is verified by "
|
|
264
|
+
f"verifying the one it copies, so it takes no model file of its own"
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
for name in required:
|
|
268
|
+
if name not in request.networks:
|
|
269
|
+
raise UsageError(
|
|
270
|
+
f"no model file was supplied for network {name!r}, which "
|
|
271
|
+
f"{model.path} declares; give {NETWORK_OPTION} {name}=<filepath> "
|
|
272
|
+
f"once for each declared network not declared equal to another"
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def check_element_types(
|
|
277
|
+
model: QueryModel, model_element_types: Mapping[str, str]
|
|
278
|
+
) -> None:
|
|
279
|
+
"""Enforce section 5.3.2's second error condition on the supplied models.
|
|
280
|
+
|
|
281
|
+
The clause is that the element types in the provided ONNX model files must
|
|
282
|
+
match the types the query declares, and that a query written in ``real`` is
|
|
283
|
+
exempt. Nothing here opens a model file, so the types the models expose
|
|
284
|
+
come from the configuration and the types they are checked against come
|
|
285
|
+
from the query. Configuration claims; the query decides.
|
|
286
|
+
|
|
287
|
+
**The exemption is a property of the whole query, not of each declaration.**
|
|
288
|
+
Section 3.5 says a real-valued query *"must exclusively use the special real
|
|
289
|
+
element type"*, so a query mixing ``real`` with any other type is not a
|
|
290
|
+
real-valued query and nothing in it is exempt - including the ``real``
|
|
291
|
+
declarations themselves. A query written entirely in ``real`` is exempt in
|
|
292
|
+
full and is not checked at all.
|
|
293
|
+
|
|
294
|
+
The parser does not enforce exclusivity: a mixed query parses cleanly with
|
|
295
|
+
both types intact, verified by running it. That is why the granularity had
|
|
296
|
+
to be decided rather than inherited. Mixing is still not refused here - this
|
|
297
|
+
clause governs whether the exemption applies, not whether the query is
|
|
298
|
+
legal, and refusing it outright would invent a rejection 5.3.2's closing
|
|
299
|
+
paragraph leaves to discretion.
|
|
300
|
+
|
|
301
|
+
A variable the configuration does not name is taken to match. The table is
|
|
302
|
+
a claim about specific variables rather than a complete description of a
|
|
303
|
+
file nobody reads, and an unnamed variable is one the configuration makes
|
|
304
|
+
no claim about. This is why it differs from the assignment values, which
|
|
305
|
+
must be complete: those are printed, and a short block would be
|
|
306
|
+
indistinguishable from a whole one, whereas nothing here reaches stdout.
|
|
307
|
+
|
|
308
|
+
Declarations are walked in the order the query makes them, so the same
|
|
309
|
+
invocation names the same variable every time.
|
|
310
|
+
|
|
311
|
+
Raises:
|
|
312
|
+
UsageError: a declared type and the model's type disagree.
|
|
313
|
+
ConfigError: the configuration names a variable the query does not
|
|
314
|
+
declare, which is the configuration author's mistake rather than
|
|
315
|
+
the caller's.
|
|
316
|
+
"""
|
|
317
|
+
if not model_element_types:
|
|
318
|
+
return
|
|
319
|
+
|
|
320
|
+
if is_real_valued(model):
|
|
321
|
+
return
|
|
322
|
+
|
|
323
|
+
declared = {declaration.name: declaration for declaration in model.declarations}
|
|
324
|
+
# Sorted, because this is a set difference and the message must not depend
|
|
325
|
+
# on iteration order. Declaration order cannot be recovered for a name the
|
|
326
|
+
# query never declared.
|
|
327
|
+
unknown = sorted(set(model_element_types) - set(declared))
|
|
328
|
+
if unknown:
|
|
329
|
+
raise ConfigError(
|
|
330
|
+
f"the rule states a model element type for {unknown[0]!r}, which "
|
|
331
|
+
f"{model.path} does not declare; a rule states what the model files "
|
|
332
|
+
f"expose, and the query decides which variables exist"
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
for declaration in model.declarations:
|
|
336
|
+
exposed = model_element_types.get(declaration.name)
|
|
337
|
+
if exposed is None or exposed == declaration.element_type:
|
|
338
|
+
continue
|
|
339
|
+
raise UsageError(
|
|
340
|
+
f"the model for network {declaration.network!r} exposes "
|
|
341
|
+
f"{declaration.name!r} as {exposed}, but {model.path} declares it "
|
|
342
|
+
f"{declaration.element_type}; a declared type and a model's type must "
|
|
343
|
+
f"agree unless the query is written entirely in {REAL}"
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def check_soundness(model: QueryModel, sound_for: Optional[Sequence[str]]) -> None:
|
|
348
|
+
"""Enforce section 5.3.2's third error condition on the declared types.
|
|
349
|
+
|
|
350
|
+
The clause is that a solver incapable of sound analysis over a declared
|
|
351
|
+
type must error, and that a user consents to unsound analysis by rewriting
|
|
352
|
+
the query to use ``real``. Which types this solver claims to analyse
|
|
353
|
+
soundly is ``[soundness] sound-for``, so the condition is triggerable on
|
|
354
|
+
demand - which is the point, because it is otherwise close to impossible to
|
|
355
|
+
provoke against a real verifier.
|
|
356
|
+
|
|
357
|
+
**Consent is given by the whole query, not by one declaration.** Section
|
|
358
|
+
3.5's real-valued query uses ``real`` exclusively, and 5.3.2 asks the user
|
|
359
|
+
to consent by *rewriting the query* - both of which describe a file rather
|
|
360
|
+
than a variable. So a query written entirely in ``real`` is not checked at
|
|
361
|
+
all, and in a query that mixes types every declaration is checked against
|
|
362
|
+
the claim, the ``real`` ones included. A configuration that wants a mixed
|
|
363
|
+
query to pass says so by listing ``real`` in ``sound-for``, which is the
|
|
364
|
+
same explicit claim every other type needs.
|
|
365
|
+
|
|
366
|
+
**A configuration with no soundness table is not checked at all**, and that
|
|
367
|
+
is a decision rather than an oversight. Read the other way, silence would
|
|
368
|
+
be a claim of total unsoundness, and every configuration that has never
|
|
369
|
+
mentioned the subject would refuse every query it was given. That is not
|
|
370
|
+
the conservative reading of an optional table, it is the most destructive
|
|
371
|
+
one. An explicitly empty ``sound-for``
|
|
372
|
+
**is** a claim, and refuses everything not written in ``real``, which is
|
|
373
|
+
exactly the fixture a caller needs to trigger this condition.
|
|
374
|
+
|
|
375
|
+
The claimed types are listed in the order the configuration writes them,
|
|
376
|
+
matching the section 5.4.1 warning, so the message is identical between
|
|
377
|
+
runs.
|
|
378
|
+
|
|
379
|
+
Raises:
|
|
380
|
+
UsageError: a declared type is one this configuration does not claim to
|
|
381
|
+
analyse soundly, and consent was not given by declaring it ``real``.
|
|
382
|
+
"""
|
|
383
|
+
if sound_for is None:
|
|
384
|
+
return
|
|
385
|
+
|
|
386
|
+
if is_real_valued(model):
|
|
387
|
+
return
|
|
388
|
+
|
|
389
|
+
claimed = set(sound_for)
|
|
390
|
+
for declaration in model.declarations:
|
|
391
|
+
if declaration.element_type in claimed:
|
|
392
|
+
continue
|
|
393
|
+
listed = ", ".join(sound_for) if sound_for else "no element type at all"
|
|
394
|
+
raise UsageError(
|
|
395
|
+
f"{model.path} declares {declaration.name!r} in network "
|
|
396
|
+
f"{declaration.network!r} as {declaration.element_type}, which this "
|
|
397
|
+
f"solver does not claim to analyse soundly; it claims {listed}. "
|
|
398
|
+
f"Rewrite the query to use {REAL} throughout to consent to analysis "
|
|
399
|
+
f"that may be unsound"
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _network_pair(token: str) -> tuple[str, Path]:
|
|
404
|
+
"""Split one --network value into the name it maps and the file it maps to.
|
|
405
|
+
|
|
406
|
+
Three ways to get this wrong - no '=' at all, no name before it, no path
|
|
407
|
+
after it - share one message, because they share one cause: the value is not
|
|
408
|
+
a pair. Splitting on the first '=' rather than the last is deliberate; a
|
|
409
|
+
network name cannot contain one, and a file name can.
|
|
410
|
+
"""
|
|
411
|
+
name, separator, path = token.partition("=")
|
|
412
|
+
if not separator or not name or not path:
|
|
413
|
+
raise UsageError(
|
|
414
|
+
f"{NETWORK_OPTION} takes <name>=<filepath>; {token!r} is not a network "
|
|
415
|
+
f"name and a file path separated by '='"
|
|
416
|
+
)
|
|
417
|
+
return name, Path(path)
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: vnnlib-test-solver
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: A configurable test solver implementing the VNN-LIB standard's command-line interface.
|
|
5
|
+
Project-URL: Homepage, https://github.com/VNNLIB/VNNLIB-Test-Solver
|
|
6
|
+
Project-URL: Repository, https://github.com/VNNLIB/VNNLIB-Test-Solver
|
|
7
|
+
Project-URL: Changelog, https://github.com/VNNLIB/VNNLIB-Test-Solver/blob/main/CHANGELOG.md
|
|
8
|
+
Project-URL: Issues, https://github.com/VNNLIB/VNNLIB-Test-Solver/issues
|
|
9
|
+
Author: VNNLIB
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: conformance,neural-network,testing,verification,vnnlib
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Testing
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Requires-Dist: tomli>=2.0; python_version < '3.11'
|
|
26
|
+
Requires-Dist: vnnlib==1.0.2
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
31
|
+
Provides-Extra: serialise
|
|
32
|
+
Requires-Dist: onnx; extra == 'serialise'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# VNNLIB-Test-Solver
|
|
36
|
+
|
|
37
|
+
[](https://github.com/VNNLIB/VNNLIB-Test-Solver/actions/workflows/buildAndTest.yml)
|
|
38
|
+
|
|
39
|
+
A configurable test solver for the [VNN-LIB](https://www.vnnlib.org/) standard. It
|
|
40
|
+
implements the command-line interface defined in Chapter 5 and answers entirely from a
|
|
41
|
+
configuration file, performing no verification of its own. It exists to give tools that
|
|
42
|
+
drive VNN-LIB solvers, in particular the `vnnlib.solver` interface, a solver whose
|
|
43
|
+
responses are fully controlled and reproducible.
|
|
44
|
+
|
|
45
|
+
Team 23, CITS3200 Professional Computing, The University of Western Australia.
|
|
46
|
+
Client: Dr Matthew Daggitt.
|
|
47
|
+
|
|
48
|
+
## Features
|
|
49
|
+
|
|
50
|
+
- Every answer comes from a TOML file. One binary gives three different solvers three
|
|
51
|
+
different behaviours with no code change.
|
|
52
|
+
- The `verify` command, returning any of `sat`, `unsat`, `unknown` or `timed-out`, chosen
|
|
53
|
+
by matching the query file's name.
|
|
54
|
+
- The satisfying assignment of section 5.3.1, printed in the command-line format and
|
|
55
|
+
compared byte for byte against the standard's own worked example, with one deliberate
|
|
56
|
+
departure where that example contradicts its own declarations. The
|
|
57
|
+
[reference](https://github.com/VNNLIB/VNNLIB-Test-Solver/blob/main/docs/CONFIGURATION.md#coverage-of-the-standards-command-line-chapter)
|
|
58
|
+
gives the four lines and why.
|
|
59
|
+
- The `supports` command, answering all eleven mandatory capabilities in the four
|
|
60
|
+
response shapes the standard defines.
|
|
61
|
+
- All three of the error conditions section 5.3.2 requires, each distinguishable from the
|
|
62
|
+
others.
|
|
63
|
+
- A configurable delay, so a caller's timeout handling can be exercised without a solver
|
|
64
|
+
that genuinely takes minutes.
|
|
65
|
+
- Four controls that make the solver misbehave on request: unparseable output, an exit
|
|
66
|
+
status disagreeing with the answer, noise on `stderr`, and abnormal termination.
|
|
67
|
+
|
|
68
|
+
**Not implemented:** `--serialise-assignments` is accepted but writes no files. That is
|
|
69
|
+
conformant rather than a gap - the standard requires the option only of a solver that
|
|
70
|
+
reports supporting it, and this one reports `false`. The
|
|
71
|
+
[coverage table](https://github.com/VNNLIB/VNNLIB-Test-Solver/blob/main/docs/CONFIGURATION.md#coverage-of-the-standards-command-line-chapter)
|
|
72
|
+
gives the clause.
|
|
73
|
+
|
|
74
|
+
## Installation
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install vnnlib-test-solver
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
To work on the package instead, install it from a clone of this repository with
|
|
81
|
+
`pip install .`, or `pip install -e ".[dev]"` to run its tests.
|
|
82
|
+
|
|
83
|
+
Python 3.9 or later. The package is pure Python; its one runtime dependency, the
|
|
84
|
+
standard's own `vnnlib` parser, is a compiled extension that publishes prebuilt wheels,
|
|
85
|
+
so on Linux `x86_64`, macOS and Windows an install needs no compiler, CMake or pybind11.
|
|
86
|
+
|
|
87
|
+
**There is no Linux `aarch64` wheel for the pinned version.** On arm64 Linux `pip install`
|
|
88
|
+
falls through to building the parser from source and does need that toolchain. It is named
|
|
89
|
+
here because it presents as a compiler error inside somebody else's package, which nobody
|
|
90
|
+
traces back to a readme. See
|
|
91
|
+
[Requirements](https://github.com/VNNLIB/VNNLIB-Test-Solver/blob/main/docs/CONFIGURATION.md#requirements) for the platform table.
|
|
92
|
+
|
|
93
|
+
## Basic Usage
|
|
94
|
+
|
|
95
|
+
**This solver has no default behaviour.** It answers only from a configuration file, and
|
|
96
|
+
running it without one is an error rather than a fallback. That applies to `--name` and
|
|
97
|
+
`--version` as much as to `verify`.
|
|
98
|
+
|
|
99
|
+
The smallest configuration that answers all four commands below:
|
|
100
|
+
|
|
101
|
+
```toml
|
|
102
|
+
[solver]
|
|
103
|
+
name = "My Test Solver"
|
|
104
|
+
version = "1.0.0"
|
|
105
|
+
|
|
106
|
+
[capabilities]
|
|
107
|
+
onnx-element-types = ["real"]
|
|
108
|
+
|
|
109
|
+
[[rules]]
|
|
110
|
+
match = "*.vnnlib"
|
|
111
|
+
result = "sat"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
vnnlibTestSolver --config my-solver.toml --name
|
|
116
|
+
vnnlibTestSolver --config my-solver.toml --version
|
|
117
|
+
vnnlibTestSolver --config my-solver.toml verify query.vnnlib
|
|
118
|
+
vnnlibTestSolver --config my-solver.toml supports --onnx-element-types
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`[solver]` and one rule are all that `verify`, `--name` and `--version` need. Each
|
|
122
|
+
capability is answered from its own key, and one the file does not mention is an error
|
|
123
|
+
rather than an empty answer, so `supports` needs the `[capabilities]` table as well.
|
|
124
|
+
|
|
125
|
+
Runnable configurations covering the four results, the capability responses, the
|
|
126
|
+
standard's worked example and the misbehaviour controls are in
|
|
127
|
+
[`examples/`](https://github.com/VNNLIB/VNNLIB-Test-Solver/tree/main/examples/).
|
|
128
|
+
|
|
129
|
+
A run resolves its configuration from `--config`, then the `VNNLIB_TEST_SOLVER_CONFIG`
|
|
130
|
+
environment variable, then a `vnnlibTestSolver.toml` in the working directory. There is no
|
|
131
|
+
fourth source and no default: if none resolves, the run exits `2` with nothing on
|
|
132
|
+
`stdout`. Every answer this solver gives is one somebody chose, so with no configuration
|
|
133
|
+
there is nothing it can honestly say.
|
|
134
|
+
|
|
135
|
+
## Documentation
|
|
136
|
+
|
|
137
|
+
- **[Configuration reference](https://github.com/VNNLIB/VNNLIB-Test-Solver/blob/main/docs/CONFIGURATION.md)** - the complete surface: rules and
|
|
138
|
+
matching, model files, element types and soundness, satisfying assignments, capability
|
|
139
|
+
queries, the misbehaviour controls, configuration discovery, exit codes, and the
|
|
140
|
+
guarantees about what reaches `stdout`. It also carries a clause-by-clause table of what
|
|
141
|
+
Chapter 5 requires and what this package does with each requirement.
|
|
142
|
+
- **[CONTRIBUTING.md](https://github.com/VNNLIB/VNNLIB-Test-Solver/blob/main/CONTRIBUTING.md)** - working on the package: setting up, running the
|
|
143
|
+
checks, how correctness is established here, and how a release is made.
|
|
144
|
+
- **[CHANGELOG.md](https://github.com/VNNLIB/VNNLIB-Test-Solver/blob/main/CHANGELOG.md)** - what changed in each version, and why.
|
|
145
|
+
|
|
146
|
+
## Output guarantees
|
|
147
|
+
|
|
148
|
+
Three properties a caller can rely on, each covered by tests that read raw bytes:
|
|
149
|
+
|
|
150
|
+
- The result is the **first line of `stdout`**, and `stdout` carries nothing else that is
|
|
151
|
+
not a response the standard defines. Warnings and errors go to `stderr`. The one
|
|
152
|
+
exception has to be asked for by name: a configuration setting `raw_stdout` replaces
|
|
153
|
+
`stdout` deliberately, which is how a caller produces output its own parser should
|
|
154
|
+
reject.
|
|
155
|
+
- Line endings on `stdout` are **`LF` on every platform**, Windows included, so a recorded
|
|
156
|
+
response compares byte for byte across machines.
|
|
157
|
+
- Output is **deterministic**: the same configuration and query produce byte-identical
|
|
158
|
+
`stdout` every time. No timestamp, duration, random value, set or hash ordering ever
|
|
159
|
+
reaches it.
|
|
160
|
+
|
|
161
|
+
## Version compatibility
|
|
162
|
+
|
|
163
|
+
| VNNLIB-Test-Solver version | VNNLIB version |
|
|
164
|
+
| --- | --- |
|
|
165
|
+
| v2.0.0 | v2.0 |
|
|
166
|
+
| v1.0.0 | v2.0 |
|
|
167
|
+
|
|
168
|
+
## Related repositories
|
|
169
|
+
|
|
170
|
+
- [VNNLIB-Standard](https://github.com/VNNLIB/VNNLIB-Standard) - the specification
|
|
171
|
+
- [VNNLIB-CPP](https://github.com/VNNLIB/VNNLIB-CPP) - the C++ parser library
|
|
172
|
+
- [VNNLIB-Python](https://github.com/VNNLIB/VNNLIB-Python) - the Python bindings
|
|
173
|
+
|
|
174
|
+
## Licence
|
|
175
|
+
|
|
176
|
+
MIT. See `LICENSE`.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
vnnlib_test_solver/__init__.py,sha256=_7OlQdbVkK4jad0CLdpI0grT-zEAb-qgFmH5mFzDXiA,22
|
|
2
|
+
vnnlib_test_solver/assignments.py,sha256=Vqr3eQ--3F2WASs9hFr7qgF6iFTjiKY2kzkQ0mB6Itc,6828
|
|
3
|
+
vnnlib_test_solver/cli.py,sha256=uSHXX0jz_UnltQE4ii2_UN-TvEyEltP66WW5VtWa7X0,20964
|
|
4
|
+
vnnlib_test_solver/config.py,sha256=mXO4qO_OTMNs38pucMwww-2o122UnJTRWHmeDyJZUh0,4287
|
|
5
|
+
vnnlib_test_solver/dtypes.py,sha256=k2DSRD3UxwUHk-EweZG3kzg97ZYsVAK_BDwcRJIUm_4,2336
|
|
6
|
+
vnnlib_test_solver/errors.py,sha256=K_l6E3UNh9FYGdA_S3U9yLbhkrdsuYTg2MOBclb8-8s,1124
|
|
7
|
+
vnnlib_test_solver/injection.py,sha256=NvW6bKT7yRjcYS7zhKEf93oIb5uM5J1cJJ4AEL3fZP8,5265
|
|
8
|
+
vnnlib_test_solver/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
vnnlib_test_solver/querymodel.py,sha256=GKWHC-HSlnIA7v8al1u4qeSZ8KUIT3XlwDKXRHumOO4,10058
|
|
10
|
+
vnnlib_test_solver/rules.py,sha256=v-GERh8D_1pVO4Y0iX6vN47excsNxE-3LHz4jqwt4tg,4520
|
|
11
|
+
vnnlib_test_solver/spec.py,sha256=kXWo8ocGA9jbAiqFThRvQ5XE3Rnn1gfiPbf_LKCENOo,2644
|
|
12
|
+
vnnlib_test_solver/supports.py,sha256=_pNvS84Fjo3AI7AU8pksFHuPoNkT6L46mCpWFAa4ugE,5716
|
|
13
|
+
vnnlib_test_solver/validation.py,sha256=IcEqV6M7qVe8hSgP-8bp1qEbANv2zjPkVMH_HA7TIv4,22180
|
|
14
|
+
vnnlib_test_solver/verify.py,sha256=2R9mptYnSkWBA7xVSy_QiytDMzVESfXtuWgKriVS-B0,18455
|
|
15
|
+
vnnlib_test_solver-2.0.0.dist-info/METADATA,sha256=CGiDhHY3FR2ghcuV3AmXMTqEYJ_5GSwt8BeTvGNRsRk,8275
|
|
16
|
+
vnnlib_test_solver-2.0.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
17
|
+
vnnlib_test_solver-2.0.0.dist-info/entry_points.txt,sha256=55Kn5rWOJ5bzFvsg4FkkTDgDg7qXlMukk9t_BrMV6dk,65
|
|
18
|
+
vnnlib_test_solver-2.0.0.dist-info/licenses/LICENSE,sha256=b5q7ljPaPaT4WAN2VLPai3V7cni78Q5xSzpgXJ5MJ10,1063
|
|
19
|
+
vnnlib_test_solver-2.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VNNLIB
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|