subsumes-benchmark 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. subsumes_benchmark-0.9.0/LICENSE +21 -0
  2. subsumes_benchmark-0.9.0/PKG-INFO +72 -0
  3. subsumes_benchmark-0.9.0/README.pypi.md +49 -0
  4. subsumes_benchmark-0.9.0/corpus/licenses/inclusion-artifact.txt +674 -0
  5. subsumes_benchmark-0.9.0/corpus/licenses/json-schema-test-suite.txt +19 -0
  6. subsumes_benchmark-0.9.0/pyproject.toml +171 -0
  7. subsumes_benchmark-0.9.0/pyproject.toml.orig +162 -0
  8. subsumes_benchmark-0.9.0/src/subsumes_benchmark/__init__.py +4 -0
  9. subsumes_benchmark-0.9.0/src/subsumes_benchmark/__main__.py +3 -0
  10. subsumes_benchmark-0.9.0/src/subsumes_benchmark/archive.py +79 -0
  11. subsumes_benchmark-0.9.0/src/subsumes_benchmark/build.py +656 -0
  12. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/__init__.py +5 -0
  13. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/__main__.py +161 -0
  14. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/checkers.py +220 -0
  15. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/derive.py +301 -0
  16. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/diff.py +364 -0
  17. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/downgrade.py +429 -0
  18. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/report.py +643 -0
  19. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/results.py +130 -0
  20. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/run.py +270 -0
  21. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/summary.py +226 -0
  22. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/tasks.py +107 -0
  23. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/witness.py +104 -0
  24. subsumes_benchmark-0.9.0/src/subsumes_benchmark/compare/words.py +43 -0
  25. subsumes_benchmark-0.9.0/src/subsumes_benchmark/config.py +32 -0
  26. subsumes_benchmark-0.9.0/src/subsumes_benchmark/convert.py +329 -0
  27. subsumes_benchmark-0.9.0/src/subsumes_benchmark/corpus.tar.zst +0 -0
  28. subsumes_benchmark-0.9.0/src/subsumes_benchmark/derived.py +82 -0
  29. subsumes_benchmark-0.9.0/src/subsumes_benchmark/features.py +225 -0
  30. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/__init__.py +1 -0
  31. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/case.py +156 -0
  32. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/checker.py +54 -0
  33. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/collections.py +93 -0
  34. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/derived.py +71 -0
  35. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/overrides.py +79 -0
  36. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/ready.py +21 -0
  37. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/reply.py +115 -0
  38. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/request.py +34 -0
  39. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/result.py +239 -0
  40. subsumes_benchmark-0.9.0/src/subsumes_benchmark/generated/run.py +111 -0
  41. subsumes_benchmark-0.9.0/src/subsumes_benchmark/testsuite.py +200 -0
  42. subsumes_benchmark-0.9.0/src/subsumes_benchmark/unfuse.py +124 -0
  43. subsumes_benchmark-0.9.0/src/subsumes_benchmark/validation.py +286 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Georg Wölflein
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,72 @@
1
+ Metadata-Version: 2.4
2
+ Name: subsumes-benchmark
3
+ Version: 0.9.0
4
+ Summary: The subsumes benchmark for JSON Schema inclusion: pairs of draft 2020-12 schemas with known answers, and the command that runs a checker over them.
5
+ Keywords: json-schema,subschema,inclusion,containment,benchmark
6
+ Author: Georg Wölflein
7
+ Author-email: Georg Wölflein <georgw7777@gmail.com>
8
+ License-Expression: MIT AND GPL-3.0-only
9
+ License-File: LICENSE
10
+ License-File: corpus/licenses/inclusion-artifact.txt
11
+ License-File: corpus/licenses/json-schema-test-suite.txt
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.14
14
+ Classifier: Typing :: Typed
15
+ Requires-Dist: jsonschema-rs==0.61.2
16
+ Requires-Dist: pydantic>=2.14.3
17
+ Requires-Dist: matplotlib>=3.10 ; extra == 'report'
18
+ Requires-Python: >=3.14
19
+ Project-URL: Repository, https://github.com/georg-wolflein/subsumes-benchmark
20
+ Project-URL: Issues, https://github.com/georg-wolflein/subsumes-benchmark/issues
21
+ Provides-Extra: report
22
+ Description-Content-Type: text/markdown
23
+
24
+ # subsumes-benchmark
25
+
26
+ The [subsumes benchmark corpus](https://github.com/georg-wolflein/subsumes-benchmark) for JSON Schema inclusion, and the command that runs a checker over it and reports on the run.
27
+
28
+ The corpus is pairs of draft 2020-12 schemas with known answers to one question: does schema A accept every instance that is valid under schema B? A checker is a program that answers it, and the [repository](https://github.com/georg-wolflein/subsumes-benchmark#comparing-checkers) compares the open-source ones.
29
+
30
+ The package of a version holds the corpus of that version, so the version pins both. Results measured on two versions are comparable exactly when their major and minor numbers match.
31
+
32
+ ## Use
33
+
34
+ The command needs Python 3.14, which [uv](https://docs.astral.sh/uv/) fetches by itself.
35
+
36
+ ```bash
37
+ # Unpack the corpus into corpus/.
38
+ uvx subsumes-benchmark==<version> corpus corpus
39
+
40
+ # Run a checker over every task; the results go to results/<folder of the checker>/.
41
+ uvx subsumes-benchmark==<version> run path/to/checker
42
+
43
+ # Hold the run against an earlier one, as a comment for a pull request.
44
+ uvx subsumes-benchmark==<version> diff results/checker --base base/checker
45
+
46
+ # Report on the runs of several checkers, with charts.
47
+ uvx --from 'subsumes-benchmark[report]==<version>' subsumes-benchmark report results
48
+ ```
49
+
50
+ - **`corpus`** unpacks the corpus into a new or empty folder: one JSON file per case, in a folder per collection. The [repository](https://github.com/georg-wolflein/subsumes-benchmark#a-case) describes a case and the collections.
51
+
52
+ - **`run`** asks the checker both directions of every case, one task after the other, and writes one line per task to `results.jsonl`. Where the checker gives a counterexample with a no, a validator is asked whether it is one. [`result.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/result.schema.json) and [`run.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/run.schema.json) describe the two files of a run.
53
+ - **`diff`** holds one run against another of the same checker. It fails when an answer is wrong, when a task ended in an error, a crash or a timeout, and when a task that the other run answered is no longer answered.
54
+ - **`report`** compares the runs of several checkers: how many tasks each one solved, how often it was wrong, and how long it took.
55
+
56
+ ## A checker
57
+
58
+ A checker is a folder with a `checker.json` and an adapter: a program that starts once, prints `{"ready": true}`, and then answers one request per line on its standard input with one line on its standard output. A request holds the two schemas and asks whether `a` accepts every instance that is valid under `b`.
59
+
60
+ ```json
61
+ {"ready": true}
62
+ {"a": {"type": "string"}, "b": {"maxLength": 3}, "timeout": 10.0}
63
+ {"answer": false, "seconds": 0.00042, "counterexample": "x"}
64
+ ```
65
+
66
+ The first line is the adapter's when it has started, the second a request, and the third the reply to it. An answer is `true`, `false` or `null`, as the label of a case is, and a reply without one says why: `{"failure": "error", "message": "…"}` or `{"failure": "timeout"}`.
67
+
68
+ [The documentation of the checkers](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/checkers/README.md) has the keys of `checker.json`, the protocol in full, and the adapters of the checkers that the repository compares.
69
+
70
+ ## Licences
71
+
72
+ The code is MIT licensed. The corpus keeps the licences of its sources, which the `collection.json` of each collection names: GPL-3.0 for the collections built from the artifact of *JSON Schema Inclusion through Refutational Normalization*, and MIT for the one built from the official JSON Schema Test Suite. The package carries the texts, and so does the corpus, under `licenses/`.
@@ -0,0 +1,49 @@
1
+ # subsumes-benchmark
2
+
3
+ The [subsumes benchmark corpus](https://github.com/georg-wolflein/subsumes-benchmark) for JSON Schema inclusion, and the command that runs a checker over it and reports on the run.
4
+
5
+ The corpus is pairs of draft 2020-12 schemas with known answers to one question: does schema A accept every instance that is valid under schema B? A checker is a program that answers it, and the [repository](https://github.com/georg-wolflein/subsumes-benchmark#comparing-checkers) compares the open-source ones.
6
+
7
+ The package of a version holds the corpus of that version, so the version pins both. Results measured on two versions are comparable exactly when their major and minor numbers match.
8
+
9
+ ## Use
10
+
11
+ The command needs Python 3.14, which [uv](https://docs.astral.sh/uv/) fetches by itself.
12
+
13
+ ```bash
14
+ # Unpack the corpus into corpus/.
15
+ uvx subsumes-benchmark==<version> corpus corpus
16
+
17
+ # Run a checker over every task; the results go to results/<folder of the checker>/.
18
+ uvx subsumes-benchmark==<version> run path/to/checker
19
+
20
+ # Hold the run against an earlier one, as a comment for a pull request.
21
+ uvx subsumes-benchmark==<version> diff results/checker --base base/checker
22
+
23
+ # Report on the runs of several checkers, with charts.
24
+ uvx --from 'subsumes-benchmark[report]==<version>' subsumes-benchmark report results
25
+ ```
26
+
27
+ - **`corpus`** unpacks the corpus into a new or empty folder: one JSON file per case, in a folder per collection. The [repository](https://github.com/georg-wolflein/subsumes-benchmark#a-case) describes a case and the collections.
28
+
29
+ - **`run`** asks the checker both directions of every case, one task after the other, and writes one line per task to `results.jsonl`. Where the checker gives a counterexample with a no, a validator is asked whether it is one. [`result.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/result.schema.json) and [`run.schema.json`](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/schemas/run.schema.json) describe the two files of a run.
30
+ - **`diff`** holds one run against another of the same checker. It fails when an answer is wrong, when a task ended in an error, a crash or a timeout, and when a task that the other run answered is no longer answered.
31
+ - **`report`** compares the runs of several checkers: how many tasks each one solved, how often it was wrong, and how long it took.
32
+
33
+ ## A checker
34
+
35
+ A checker is a folder with a `checker.json` and an adapter: a program that starts once, prints `{"ready": true}`, and then answers one request per line on its standard input with one line on its standard output. A request holds the two schemas and asks whether `a` accepts every instance that is valid under `b`.
36
+
37
+ ```json
38
+ {"ready": true}
39
+ {"a": {"type": "string"}, "b": {"maxLength": 3}, "timeout": 10.0}
40
+ {"answer": false, "seconds": 0.00042, "counterexample": "x"}
41
+ ```
42
+
43
+ The first line is the adapter's when it has started, the second a request, and the third the reply to it. An answer is `true`, `false` or `null`, as the label of a case is, and a reply without one says why: `{"failure": "error", "message": "…"}` or `{"failure": "timeout"}`.
44
+
45
+ [The documentation of the checkers](https://github.com/georg-wolflein/subsumes-benchmark/blob/main/checkers/README.md) has the keys of `checker.json`, the protocol in full, and the adapters of the checkers that the repository compares.
46
+
47
+ ## Licences
48
+
49
+ The code is MIT licensed. The corpus keeps the licences of its sources, which the `collection.json` of each collection names: GPL-3.0 for the collections built from the artifact of *JSON Schema Inclusion through Refutational Normalization*, and MIT for the one built from the official JSON Schema Test Suite. The package carries the texts, and so does the corpus, under `licenses/`.