subsumes-benchmark 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- subsumes_benchmark-0.9.0/LICENSE +21 -0
- subsumes_benchmark-0.9.0/PKG-INFO +72 -0
- subsumes_benchmark-0.9.0/README.pypi.md +49 -0
- subsumes_benchmark-0.9.0/corpus/licenses/inclusion-artifact.txt +674 -0
- subsumes_benchmark-0.9.0/corpus/licenses/json-schema-test-suite.txt +19 -0
- subsumes_benchmark-0.9.0/pyproject.toml +171 -0
- subsumes_benchmark-0.9.0/pyproject.toml.orig +162 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/__init__.py +4 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/__main__.py +3 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/archive.py +79 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/build.py +656 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/__init__.py +5 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/__main__.py +161 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/checkers.py +220 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/derive.py +301 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/diff.py +364 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/downgrade.py +429 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/report.py +643 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/results.py +130 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/run.py +270 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/summary.py +226 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/tasks.py +107 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/witness.py +104 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/words.py +43 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/config.py +32 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/convert.py +329 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/corpus.tar.zst +0 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/derived.py +82 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/features.py +225 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/__init__.py +1 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/case.py +156 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/checker.py +54 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/collections.py +93 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/derived.py +71 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/overrides.py +79 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/ready.py +21 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/reply.py +115 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/request.py +34 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/result.py +239 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/run.py +111 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/testsuite.py +200 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/unfuse.py +124 -0
- subsumes_benchmark-0.9.0/src/subsumes_benchmark/validation.py +286 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Georg Wölflein
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: subsumes-benchmark
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: The subsumes benchmark for JSON Schema inclusion: pairs of draft 2020-12 schemas with known answers, and the command that runs a checker over them.
|
|
5
|
+
Keywords: json-schema,subschema,inclusion,containment,benchmark
|
|
6
|
+
Author: Georg Wölflein
|
|
7
|
+
Author-email: Georg Wölflein <georgw7777@gmail.com>
|
|
8
|
+
License-Expression: MIT AND GPL-3.0-only
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
License-File: corpus/licenses/inclusion-artifact.txt
|
|
11
|
+
License-File: corpus/licenses/json-schema-test-suite.txt
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: jsonschema-rs==0.61.2
|
|
16
|
+
Requires-Dist: pydantic>=2.14.3
|
|
17
|
+
Requires-Dist: matplotlib>=3.10 ; extra == 'report'
|
|
18
|
+
Requires-Python: >=3.14
|
|
19
|
+
Project-URL: Repository, https://github.com/georg-wolflein/subsumes-benchmark
|
|
20
|
+
Project-URL: Issues, https://github.com/georg-wolflein/subsumes-benchmark/issues
|
|
21
|
+
Provides-Extra: report
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# subsumes-benchmark
|
|
25
|
+
|
|
26
|
+
The [subsumes benchmark corpus](https://github.com/georg-wolflein/subsumes-benchmark) for JSON Schema inclusion, and the command that runs a checker over it and reports on the run.
|
|
27
|
+
|
|
28
|
+
The corpus is pairs of draft 2020-12 schemas with known answers to one question: does schema A accept every instance that is valid under schema B? A checker is a program that answers it, and the [repository](https://github.com/georg-wolflein/subsumes-benchmark#comparing-checkers) compares the open-source ones.
|
|
29
|
+
|
|
30
|
+
The package of a version holds the corpus of that version, so the version pins both. Results measured on two versions are comparable exactly when their major and minor numbers match.
|
|
31
|
+
|
|
32
|
+
## Use
|
|
33
|
+
|
|
34
|
+
The command needs Python 3.14, which [uv](https://docs.astral.sh/uv/) fetches by itself.
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
# Unpack the corpus into corpus/.
|
|
38
|
+
uvx subsumes-benchmark==<version> corpus corpus
|
|
39
|
+
|
|
40
|
+
# Run a checker over every task; the results go to results/<folder of the checker>/.
|
|
41
|
+
uvx subsumes-benchmark==<version> run path/to/checker
|
|
42
|
+
|
|
43
|
+
# Hold the run against an earlier one, as a comment for a pull request.
|
|
44
|
+
uvx subsumes-benchmark==<version> diff results/checker --base base/checker
|
|
45
|
+
|
|
46
|
+
# Report on the runs of several checkers, with charts.
|
|
47
|
+
uvx --from 'subsumes-benchmark[report]==<version>' subsumes-benchmark report results
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
- **`corpus`** unpacks the corpus into a new or empty folder: one JSON file per case, in a folder per collection. The [repository](https://github.com/georg-wolflein/subsumes-benchmark#a-case) describes a case and the collections.
|
|
51
|
+
|
|
52
|
+
- **`run`** asks the checker both directions of every case, one task after the other, and writes one line per task to `results.jsonl`. Where the checker gives a counterexample with a no, a validator is asked whether it is one. [`result.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/result.schema.json) and [`run.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/run.schema.json) describe the two files of a run.
|
|
53
|
+
- **`diff`** holds one run against another of the same checker. It fails when an answer is wrong, when a task ended in an error, a crash or a timeout, and when a task that the other run answered is no longer answered.
|
|
54
|
+
- **`report`** compares the runs of several checkers: how many tasks each one solved, how often it was wrong, and how long it took.
|
|
55
|
+
|
|
56
|
+
## A checker
|
|
57
|
+
|
|
58
|
+
A checker is a folder with a `checker.json` and an adapter: a program that starts once, prints `{"ready": true}`, and then answers one request per line on its standard input with one line on its standard output. A request holds the two schemas and asks whether `a` accepts every instance that is valid under `b`.
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
{"ready": true}
|
|
62
|
+
{"a": {"type": "string"}, "b": {"maxLength": 3}, "timeout": 10.0}
|
|
63
|
+
{"answer": false, "seconds": 0.00042, "counterexample": "x"}
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The first line is the adapter's when it has started, the second a request, and the third the reply to it. An answer is `true`, `false` or `null`, as the label of a case is, and a reply without one says why: `{"failure": "error", "message": "…"}` or `{"failure": "timeout"}`.
|
|
67
|
+
|
|
68
|
+
[The documentation of the checkers](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/checkers/README.md) has the keys of `checker.json`, the protocol in full, and the adapters of the checkers that the repository compares.
|
|
69
|
+
|
|
70
|
+
## Licences
|
|
71
|
+
|
|
72
|
+
The code is MIT licensed. The corpus keeps the licences of its sources, which the `collection.json` of each collection names: GPL-3.0 for the collections built from the artifact of *JSON Schema Inclusion through Refutational Normalization*, and MIT for the one built from the official JSON Schema Test Suite. The package carries the texts, and so does the corpus, under `licenses/`.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# subsumes-benchmark
|
|
2
|
+
|
|
3
|
+
The [subsumes benchmark corpus](https://github.com/georg-wolflein/subsumes-benchmark) for JSON Schema inclusion, and the command that runs a checker over it and reports on the run.
|
|
4
|
+
|
|
5
|
+
The corpus is pairs of draft 2020-12 schemas with known answers to one question: does schema A accept every instance that is valid under schema B? A checker is a program that answers it, and the [repository](https://github.com/georg-wolflein/subsumes-benchmark#comparing-checkers) compares the open-source ones.
|
|
6
|
+
|
|
7
|
+
The package of a version holds the corpus of that version, so the version pins both. Results measured on two versions are comparable exactly when their major and minor numbers match.
|
|
8
|
+
|
|
9
|
+
## Use
|
|
10
|
+
|
|
11
|
+
The command needs Python 3.14, which [uv](https://docs.astral.sh/uv/) fetches by itself.
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
# Unpack the corpus into corpus/.
|
|
15
|
+
uvx subsumes-benchmark==<version> corpus corpus
|
|
16
|
+
|
|
17
|
+
# Run a checker over every task; the results go to results/<folder of the checker>/.
|
|
18
|
+
uvx subsumes-benchmark==<version> run path/to/checker
|
|
19
|
+
|
|
20
|
+
# Hold the run against an earlier one, as a comment for a pull request.
|
|
21
|
+
uvx subsumes-benchmark==<version> diff results/checker --base base/checker
|
|
22
|
+
|
|
23
|
+
# Report on the runs of several checkers, with charts.
|
|
24
|
+
uvx --from 'subsumes-benchmark[report]==<version>' subsumes-benchmark report results
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
- **`corpus`** unpacks the corpus into a new or empty folder: one JSON file per case, in a folder per collection. The [repository](https://github.com/georg-wolflein/subsumes-benchmark#a-case) describes a case and the collections.
|
|
28
|
+
|
|
29
|
+
- **`run`** asks the checker both directions of every case, one task after the other, and writes one line per task to `results.jsonl`. Where the checker gives a counterexample with a no, a validator is asked whether it is one. [`result.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/result.schema.json) and [`run.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/run.schema.json) describe the two files of a run.
|
|
30
|
+
- **`diff`** holds one run against another of the same checker. It fails when an answer is wrong, when a task ended in an error, a crash or a timeout, and when a task that the other run answered is no longer answered.
|
|
31
|
+
- **`report`** compares the runs of several checkers: how many tasks each one solved, how often it was wrong, and how long it took.
|
|
32
|
+
|
|
33
|
+
## A checker
|
|
34
|
+
|
|
35
|
+
A checker is a folder with a `checker.json` and an adapter: a program that starts once, prints `{"ready": true}`, and then answers one request per line on its standard input with one line on its standard output. A request holds the two schemas and asks whether `a` accepts every instance that is valid under `b`.
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{"ready": true}
|
|
39
|
+
{"a": {"type": "string"}, "b": {"maxLength": 3}, "timeout": 10.0}
|
|
40
|
+
{"answer": false, "seconds": 0.00042, "counterexample": "x"}
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The first line is the adapter's when it has started, the second a request, and the third the reply to it. An answer is `true`, `false` or `null`, as the label of a case is, and a reply without one says why: `{"failure": "error", "message": "…"}` or `{"failure": "timeout"}`.
|
|
44
|
+
|
|
45
|
+
[The documentation of the checkers](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/checkers/README.md) has the keys of `checker.json`, the protocol in full, and the adapters of the checkers that the repository compares.
|
|
46
|
+
|
|
47
|
+
## Licences
|
|
48
|
+
|
|
49
|
+
The code is MIT licensed. The corpus keeps the licences of its sources, which the `collection.json` of each collection names: GPL-3.0 for the collections built from the artifact of *JSON Schema Inclusion through Refutational Normalization*, and MIT for the one built from the official JSON Schema Test Suite. The package carries the texts, and so does the corpus, under `licenses/`.
|