infer-check 0.1.0__tar.gz

infer_check-0.1.0/.github/workflows/coverage.yml

@@ -0,0 +1,36 @@
+name: Run tests and upload coverage
+
+on: push
+
+jobs:
+  test:
+    name: Run tests and collect coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          activate-environment: "true"
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv pip install -e .[dev]
+
+      - name: Run tests with coverage
+        run: uv run pytest --cov=src --cov-branch --cov-report=xml
+
+      - name: Run Mypy
+        run: uv run mypy src --cobertura-xml-report mypy_report
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.xml, ./mypy_report/cobertura.xml
+          flags: pytest,mypy
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: ${{ github.repository }}

infer_check-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,70 @@
+name: Release to PyPI
+
+on:
+    push:
+        tags:
+            - "v*"
+
+jobs:
+    build:
+        name: Build distribution
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v6
+            - name: Set up uv
+              uses: astral-sh/setup-uv@v7
+              with:
+                  activate-environment: "true"
+                  python-version: "3.13"
+
+            - name: Install build tools
+              run: uv pip install build
+
+            - name: Build sdist and wheel
+              run: uv build
+
+            - name: Store the distribution packages
+              uses: actions/upload-artifact@v6
+              with:
+                  name: infer-check
+                  path: dist/
+
+    publish-to-testpypi:
+        name: Publish to TestPyPI
+        needs: build
+        runs-on: ubuntu-latest
+        environment:
+            name: testpypi
+            url: https://test.pypi.org
+        permissions:
+            id-token: write
+        steps:
+            - name: Download all the dists
+              uses: actions/download-artifact@v6
+              with:
+                  name: infer-check
+                  path: dist/
+
+            - name: Publish to TestPyPI
+              uses: pypa/gh-action-pypi-publish@release/v1
+              with:
+                  repository-url: https://test.pypi.org/legacy/
+
+    publish-to-pypi:
+        name: Publish to PyPI
+        needs: publish-to-testpypi
+        runs-on: ubuntu-latest
+        environment:
+            name: pypi
+            url: https://pypi.org
+        permissions:
+            id-token: write
+        steps:
+            - name: Download all the dists
+              uses: actions/download-artifact@v6
+              with:
+                  name: infer-check
+                  path: dist/
+
+            - name: Publish to TestPyPI
+              uses: pypa/gh-action-pypi-publish@release/v1

infer_check-0.1.0/.gitignore

@@ -0,0 +1,62 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# uv
+uv.lock
+
+# Type checking / linting / testing
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.coverage
+htmlcov/
+.test_cache/
+
+# infer-check results and caches (generated)
+results/
+*.report.html
+*.report.json
+.infer_check_cache/
+
+# Model weights (never commit)
+*.safetensors
+*.gguf
+*.bin
+*.pt
+*.pth
+*.onnx
+*.mlpackage
+
+# mlx model cache
+~/.cache/huggingface/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Secrets
+.env
+.env.local
+
+# Internal planning
+ROADMAP.md

infer_check-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,66 @@
+repos:
+  # General file hygiene
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+      - id: check-toml
+      - id: check-added-large-files
+        args: ["--maxkb=500"]
+      - id: check-merge-conflict
+      - id: detect-private-key
+
+  # Ruff — lint + format in one pass
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.4
+    hooks:
+      - id: ruff
+        args: ["--fix", "--exit-non-zero-on-fix"]
+        types_or: [python, pyi]
+      - id: ruff-format
+        types_or: [python, pyi]
+
+  # Type checking
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.15.0
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - pydantic>=2.5.0
+          - numpy>=1.26.0
+          - click>=8.1.0
+          - httpx>=0.27.0
+          - types-jinja2
+        args: ["--strict", "--ignore-missing-imports"]
+        pass_filenames: false
+        entry: mypy src/infer_check/
+
+  # Validate JSONL prompt suites
+  - repo: local
+    hooks:
+      - id: validate-prompt-suites
+        name: validate prompt suite JSONL
+        entry: python
+        args:
+          - -c
+          - |
+            import json, sys, pathlib
+            errors = []
+            for f in pathlib.Path('prompt-suites').glob('*.jsonl'):
+                for i, line in enumerate(f.read_text().splitlines(), 1):
+                    if not line.strip():
+                        continue
+                    try:
+                        obj = json.loads(line)
+                        assert 'text' in obj, 'missing text field'
+                    except Exception as e:
+                        errors.append(f'{f}:{i}: {e}')
+            if errors:
+                print('\n'.join(errors), file=sys.stderr)
+                sys.exit(1)
+        language: python
+        files: 'prompt-suites/.*\.jsonl$'
+        pass_filenames: false

infer_check-0.1.0/CHANGELOG.md

@@ -0,0 +1,9 @@
+# 0.1.0 (2026-03-11)
+
+- Initial release
+- Commands: sweep, diff, stress, determinism, report
+- Backends: mlx-lm, llama.cpp, vllm-mlx, openai-compat
+- Baseline self-check (run-twice determinism validation)
+- Severity tiers: identical / minor / moderate / severe
+- Chat template auto-detection for instruct models
+- Comprehensive error handling across all backends

infer_check-0.1.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Karan Kunal Mohindroo
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

infer_check-0.1.0/Makefile

@@ -0,0 +1,29 @@
+.PHONY: install dev lint format test test-unit test-integration clean
+
+install:
+	uv sync
+
+dev:
+	uv sync --all-extras
+	uv run pre-commit install
+
+lint:
+	uv run ruff check src/ tests/
+
+format:
+	uv run ruff format src/ tests/
+
+typecheck:
+	uv run mypy src/infer_check/
+
+test: test-unit
+
+test-unit:
+	uv run pytest tests/unit -v --tb=short
+
+test-integration:
+	uv run pytest tests/integration -v --tb=short
+
+clean:
+	rm -rf build/ dist/ *.egg-info .mypy_cache .ruff_cache .pytest_cache results/
+	find . -type d -name __pycache__ -exec rm -rf {} +

infer_check-0.1.0/PKG-INFO

@@ -0,0 +1,210 @@
+Metadata-Version: 2.4
+Name: infer-check
+Version: 0.1.0
+Summary: Correctness and reliability testing for LLM inference engines
+Project-URL: Repository, https://github.com/nullpointerdepressivedisorder/infer-check
+Author-email: Karan Kunal Mohindroo <karan@karankm.com>
+License: Apache-2.0
+License-File: LICENSE
+Keywords: correctness,llm,quantization,reliability,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: click>=8.3.1
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: numpy>=2.4.3
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: rich>=14.3.3
+Requires-Dist: scipy>=1.17.1
+Requires-Dist: tqdm>=4.67.3
+Provides-Extra: all
+Requires-Dist: matplotlib>=3.10.8; extra == 'all'
+Requires-Dist: mlx-lm<0.31.0; extra == 'all'
+Requires-Dist: mlx>=0.31.0; extra == 'all'
+Requires-Dist: scikit-learn>=1.8.0; extra == 'all'
+Provides-Extra: analysis
+Requires-Dist: matplotlib>=3.10.8; extra == 'analysis'
+Requires-Dist: scikit-learn>=1.8.0; extra == 'analysis'
+Provides-Extra: dev
+Requires-Dist: matplotlib>=3.10.8; extra == 'dev'
+Requires-Dist: mlx-lm<0.31.0; extra == 'dev'
+Requires-Dist: mlx>=0.31.0; extra == 'dev'
+Requires-Dist: mypy>=1.19.1; extra == 'dev'
+Requires-Dist: pytest-cov>=7.0.0; extra == 'dev'
+Requires-Dist: pytest>=9.0.2; extra == 'dev'
+Requires-Dist: ruff>=0.15.5; extra == 'dev'
+Requires-Dist: scikit-learn>=1.8.0; extra == 'dev'
+Provides-Extra: mlx
+Requires-Dist: mlx-lm<0.31.0; extra == 'mlx'
+Requires-Dist: mlx>=0.31.0; extra == 'mlx'
+Description-Content-Type: text/markdown
+
+# infer-check
+
+**Correctness and reliability testing for LLM inference engines.**
+
+`infer-check` is a CLI tool that tests whether LLM inference backends produce correct, stable, and deterministic output. It catches the bugs that benchmarks miss — quantization-induced failures, cross-backend divergence, KV cache corruption under load, and non-determinism at temperature=0.
+
+## Key findings
+
+Tested across Llama-3.1-8B-Instruct and Qwen3.5-4B (MoE) on Apple Silicon using mlx-lm and vllm-mlx.
+
+**4-bit quantization degrades task-dependently.** Numerical tasks break worst:
+
+```
+                       Llama-3.1-8B: bf16 vs 4bit
+┏━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
+┃ prompt suite          ┃ identical ┃ severe   ┃ mean_similarity ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
+│ adversarial-numerics  │     0/30  │   23/30  │          0.311  │
+│ reasoning             │     1/50  │   35/50  │          0.384  │
+│ code                  │     0/49  │   30/49  │          0.452  │
+└───────────────────────┴───────────┴──────────┴─────────────────┘
+```
+
+**Dense and MoE architectures degrade similarly at 4-bit.** Qwen3.5-4B (Gated Delta Networks + sparse MoE) shows 35/50 severe on reasoning — the same rate as dense Llama-3.1-8B.
+
+**vllm-mlx's serving layer is perfectly faithful.** mlx-lm vs vllm-mlx at temperature=0 on Llama-3.1-8B-4bit: 50/50 identical (reasoning) and 30/30 identical (numerics). The serving layer introduces zero divergence.
+
+**Both engines are deterministic at temperature=0.** Llama-3.1-8B-4bit and Qwen3.5-4B both scored 50/50 perfect determinism across 20 runs per prompt.
+
+**vllm-mlx handles concurrent load without corruption.** Stress test at concurrency 1/2/4/8: zero errors, 100% output consistency at all levels.
+
+## Installation
+
+```bash
+pip install infer-check
+
+# With MLX backend support (Apple Silicon)
+pip install "infer-check[mlx]"
+```
+
+## Usage
+
+### Quantization sweep
+
+Compare pre-quantized models against a baseline. Each model is a separate HuggingFace repo.
+
+```bash
+infer-check sweep \
+  --models "bf16=mlx-community/Meta-Llama-3.1-8B-Instruct-bf16,\
+            8bit=mlx-community/Meta-Llama-3.1-8B-Instruct-8bit,\
+            4bit=mlx-community/Meta-Llama-3.1-8B-Instruct-4bit" \
+  --backend mlx-lm \
+  --prompts reasoning \
+  --output ./results/sweep/
+```
+
+`--prompts` accepts either a bundled suite name (`reasoning`, `code`, `adversarial-numerics`, `determinism`, `long-context`) or a path to any `.jsonl` file.
+
+The baseline is automatically run twice as a self-check — if it's not 50/50 identical, your comparison data is unreliable.
+
+```
+                                 Sweep Summary
+┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
+┃ quant_level         ┃ identical ┃ minor ┃ moderate ┃ severe ┃ mean_similarity ┃
+┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
+│ bf16 (self-check)   │     50/50 │  0/50 │     0/50 │   0/50 │          1.0000 │
+│ 8bit                │     20/50 │  9/50 │    12/50 │   9/50 │          0.8067 │
+│ 4bit                │      1/50 │  3/50 │    11/50 │  35/50 │          0.3837 │
+└─────────────────────┴───────────┴───────┴──────────┴────────┴─────────────────┘
+```
+
+### Cross-backend diff
+
+Same model, same quant, different inference paths. Catches serving-layer bugs.
+
+```bash
+# Start vllm-mlx in another terminal:
+# vllm-mlx serve mlx-community/Meta-Llama-3.1-8B-Instruct-4bit --port 8000
+
+infer-check diff \
+  --model mlx-community/Meta-Llama-3.1-8B-Instruct-4bit \
+  --backends "mlx-lm,openai-compat" \
+  --base-urls ",http://localhost:8000" \
+  --prompts reasoning \
+  --output ./results/diff/
+```
+
+Uses `/v1/chat/completions` by default (`--chat`) so server-side chat templates match the local backend. Pass `--no-chat` for raw `/v1/completions`.
+
+### Determinism
+
+Same prompt N times at temperature=0. Output should be bit-identical every run.
+
+```bash
+infer-check determinism \
+  --model mlx-community/Meta-Llama-3.1-8B-Instruct-4bit \
+  --backend mlx-lm \
+  --prompts determinism \
+  --runs 20 \
+  --output ./results/determinism/
+```
+
+### Stress test
+
+Concurrent requests through a serving backend. Tests KV cache correctness under load.
+
+```bash
+infer-check stress \
+  --model mlx-community/Meta-Llama-3.1-8B-Instruct-4bit \
+  --backend openai-compat \
+  --base-url http://localhost:8000 \
+  --prompts reasoning \
+  --concurrency 1,2,4,8 \
+  --output ./results/stress/
+```
+
+### Report
+
+Generate an HTML report from all saved results.
+
+```bash
+infer-check report ./results/ --format html
+```
+
+## Prompt suites
+
+Curated prompts targeting known quantization failure modes:
+
+| Suite | Count | Purpose |
+|---|---|---|
+| `reasoning.jsonl` | 50 | Multi-step math and logic |
+| `code.jsonl` | 49 | Python, JSON, SQL generation |
+| `adversarial-numerics.jsonl` | 30 | IEEE 754 edge cases, overflow, precision |
+| `long-context.jsonl` | 10 | Tables and transcripts with recall questions |
+| `determinism.jsonl` | 50 | High-entropy continuations for determinism testing |
+
+All suites ship with the package — no need to clone the repo. Custom suites are JSONL files: `{"id": "...", "text": "...", "category": "...", "max_tokens": N}` per line.
+
+## Supported backends
+
+| Backend | Type | Use case |
+|---|---|---|
+| **mlx-lm** | In-process | Local Apple Silicon inference with logprobs |
+| **llama.cpp** | HTTP | `llama-server` via `/completion` endpoint |
+| **vllm-mlx** | HTTP | Continuous batching on Apple Silicon |
+| **openai-compat** | HTTP | Any OpenAI-compatible server (vLLM, SGLang, Ollama) |
+
+## Why this exists
+
+Every LLM inference engine has correctness bugs that benchmarks don't catch:
+
+- **KV cache NaN pollution** in vLLM-Ascend permanently corrupts all subsequent requests
+- **FP8 KV quantization** in vLLM causes repeated garbage output
+- **32.5% element mismatches** in SGLang's FP8 DeepGEMM kernels on Blackwell GPUs
+- **Batch-size-dependent output** where tokens change depending on concurrent request count
+
+These aren't model quality problems — they're engine correctness failures. Benchmarks like lm-evaluation-harness test whether models are smart. `infer-check` tests whether engines are correct.
+
+## Requirements
+
+- Python >= 3.11
+- macOS with Apple Silicon (for mlx-lm backend) or Linux
+- At least one backend installed
+
+## License
+
+Apache 2.0