gguf-preflight 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gguf_preflight-0.1.0/.gitignore +10 -0
- gguf_preflight-0.1.0/LICENSE +21 -0
- gguf_preflight-0.1.0/PKG-INFO +166 -0
- gguf_preflight-0.1.0/README.md +140 -0
- gguf_preflight-0.1.0/pyproject.toml +65 -0
- gguf_preflight-0.1.0/src/gguf_preflight/__init__.py +3 -0
- gguf_preflight-0.1.0/src/gguf_preflight/adapters/__init__.py +28 -0
- gguf_preflight-0.1.0/src/gguf_preflight/adapters/base.py +563 -0
- gguf_preflight-0.1.0/src/gguf_preflight/adapters/llama_server.py +46 -0
- gguf_preflight-0.1.0/src/gguf_preflight/adapters/lmstudio.py +67 -0
- gguf_preflight-0.1.0/src/gguf_preflight/adapters/ollama.py +70 -0
- gguf_preflight-0.1.0/src/gguf_preflight/cli.py +262 -0
- gguf_preflight-0.1.0/src/gguf_preflight/inspector/__init__.py +1 -0
- gguf_preflight-0.1.0/src/gguf_preflight/inspector/gguf_header.py +615 -0
- gguf_preflight-0.1.0/src/gguf_preflight/inspector/hardware.py +157 -0
- gguf_preflight-0.1.0/src/gguf_preflight/mcp_server.py +87 -0
- gguf_preflight-0.1.0/src/gguf_preflight/models.py +155 -0
- gguf_preflight-0.1.0/src/gguf_preflight/probes/__init__.py +13 -0
- gguf_preflight-0.1.0/src/gguf_preflight/probes/agent_smoke.py +549 -0
- gguf_preflight-0.1.0/src/gguf_preflight/probes/speed_probe.py +186 -0
- gguf_preflight-0.1.0/src/gguf_preflight/probes/wire_probe.py +496 -0
- gguf_preflight-0.1.0/src/gguf_preflight/py.typed +0 -0
- gguf_preflight-0.1.0/src/gguf_preflight/registry/__init__.py +1 -0
- gguf_preflight-0.1.0/src/gguf_preflight/registry/receipt.py +199 -0
- gguf_preflight-0.1.0/src/gguf_preflight/remediate/__init__.py +1 -0
- gguf_preflight-0.1.0/src/gguf_preflight/remediate/rules.py +326 -0
- gguf_preflight-0.1.0/src/gguf_preflight/runner.py +225 -0
- gguf_preflight-0.1.0/suites/core_code_10.yaml +139 -0
- gguf_preflight-0.1.0/suites/core_context_4.yaml +43 -0
- gguf_preflight-0.1.0/suites/core_tools_8.yaml +170 -0
- gguf_preflight-0.1.0/tests/conftest.py +4 -0
- gguf_preflight-0.1.0/tests/fixtures/ollama_anth_tool.json +26 -0
- gguf_preflight-0.1.0/tests/fixtures/ollama_anth_tool_stream.sse +29 -0
- gguf_preflight-0.1.0/tests/fixtures/ollama_oai_plain.json +26 -0
- gguf_preflight-0.1.0/tests/fixtures/ollama_oai_tool.json +37 -0
- gguf_preflight-0.1.0/tests/fixtures/ollama_oai_tool_stream.sse +95 -0
- gguf_preflight-0.1.0/tests/fixtures/ollama_ps.json +1 -0
- gguf_preflight-0.1.0/tests/gguf_fixtures.py +74 -0
- gguf_preflight-0.1.0/tests/test_agent_smoke.py +522 -0
- gguf_preflight-0.1.0/tests/test_gguf_header.py +266 -0
- gguf_preflight-0.1.0/tests/test_mcp_server.py +443 -0
- gguf_preflight-0.1.0/tests/test_remediate.py +96 -0
- gguf_preflight-0.1.0/tests/test_wire_probe.py +582 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 sebas8866
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: gguf-preflight
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deterministic preflight checks for local GGUF inference stacks before you connect an AI coding agent.
|
|
5
|
+
Project-URL: Repository, https://github.com/sebas8866/gguf-preflight
|
|
6
|
+
Author: sebas8866
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: ai-agents,gguf,llama.cpp,lm-studio,local-llm,mcp,ollama
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Requires-Dist: fastmcp>=4.2.4
|
|
20
|
+
Requires-Dist: httpx>=0.28.1
|
|
21
|
+
Requires-Dist: pydantic>=2.0
|
|
22
|
+
Requires-Dist: pyyaml>=6.0.4
|
|
23
|
+
Requires-Dist: rich>=15.0.0
|
|
24
|
+
Requires-Dist: typer>=0.29.2
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# gguf-preflight
|
|
28
|
+
|
|
29
|
+
Check a local GGUF model stack before you point an AI coding agent at it.
|
|
30
|
+
|
|
31
|
+
A local model can look healthy in a chat window and still break an agent. The server may silently cut off your context, drop tool calls, miss the prompt cache on every turn, or run a quant that writes fluent, broken code. `gguf-preflight` tests for those failures, gives a verdict, and prints the exact fix for your runtime.
|
|
32
|
+
|
|
33
|
+
- Works with **Ollama**, **llama-server**, **LM Studio**, and other local OpenAI- or Anthropic-compatible servers.
|
|
34
|
+
- Ships as a **CLI**, a **Python library**, and an **MCP server**.
|
|
35
|
+
- No account, no telemetry, no GPU kernel code. MIT licensed.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
You need [uv](https://docs.astral.sh/uv/) and a running local server.
|
|
40
|
+
|
|
41
|
+
```powershell
|
|
42
|
+
uvx gguf-preflight check http://localhost:11434 --model qwen3:8b --agent claude-code
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Inspect any GGUF without downloading its weights:
|
|
46
|
+
|
|
47
|
+
```powershell
|
|
48
|
+
uvx gguf-preflight inspect https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/blob/main/tinyllama-1.1b-chat-v1.0.Q2_K.gguf
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## What it checks
|
|
52
|
+
|
|
53
|
+
| Layer | Check | Failure code |
|
|
54
|
+
|---|---|---|
|
|
55
|
+
| GGUF header | Missing tokenizer, pre-tokenizer, or chat template. Low-bit quant with no importance matrix | `METADATA_DEFECT` |
|
|
56
|
+
| Wire | Silent context truncation at 4k and 8k (16k with `--deep`) | `WIRE_BUG` |
|
|
57
|
+
| Wire | Tool calls in streaming and non-streaming mode: empty drops, plaintext leaks, invalid JSON | `WIRE_BUG` |
|
|
58
|
+
| Cache | Whether an identical prefix is reused on the next request | `PREFIX_CACHE_MISS` |
|
|
59
|
+
| Agent smoke | 10 coding tasks run against hidden tests | `QUANT_COLLAPSE` |
|
|
60
|
+
| Agent smoke | 8 strict tool tasks: right tool, types, arguments, forbidden tools | `QUANT_COLLAPSE` |
|
|
61
|
+
| Agent smoke | 4 multi-turn tasks: does a turn 1 rule survive to turn 4 | `QUANT_COLLAPSE` |
|
|
62
|
+
| Speed | TTFT, prompt tok/s, generation tok/s, cold vs warm prefill (median and IQR) | `LATENCY_BOTTLENECK` |
|
|
63
|
+
|
|
64
|
+
`QUANT_COLLAPSE` means the stack answers fluently but scores below a suite's failure threshold. That can be a damaged quant. It can also be a model that is simply too small for the tasks. Borderline scores are warnings and do not blame the quant.
|
|
65
|
+
|
|
66
|
+
## Verdicts and exit codes
|
|
67
|
+
|
|
68
|
+
| Verdict | Meaning | Exit code |
|
|
69
|
+
|---|---|---|
|
|
70
|
+
| `GO` | Nothing failed | 0 |
|
|
71
|
+
| `CAUTION` | Usable, with warnings | 0 |
|
|
72
|
+
| `NO_GO` | A check failed | 1 |
|
|
73
|
+
| `INCOMPLETE` | A core check could not run, for example the GPU ran out of memory | 2 |
|
|
74
|
+
|
|
75
|
+
## Commands
|
|
76
|
+
|
|
77
|
+
```text
|
|
78
|
+
gguf-preflight check <endpoint> --model <name> [--gguf <path-or-url>]
|
|
79
|
+
[--agent claude-code|cline|generic] [--json] [--out receipt.json]
|
|
80
|
+
[--deep] [--allow-remote]
|
|
81
|
+
gguf-preflight inspect <path-or-hf-url> [--json]
|
|
82
|
+
gguf-preflight diff <receipt_a.json> <receipt_b.json> [--json] [--fail-on-regression]
|
|
83
|
+
gguf-preflight mcp
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Public hosts are refused unless you pass `--allow-remote`. If your server needs a key, set `GGUF_PREFLIGHT_API_KEY`. It is never written to a receipt. Set `HF_TOKEN` for gated Hugging Face repos.
|
|
87
|
+
|
|
88
|
+
## Receipts
|
|
89
|
+
|
|
90
|
+
`--out` saves a JSON receipt with the verdict, GGUF metadata, runtime, hardware, every probe result, the fixes, and a hash of the test transcript. Compare two receipts after you change a quant, a flag, or a runtime version:
|
|
91
|
+
|
|
92
|
+
```powershell
|
|
93
|
+
gguf-preflight check http://localhost:11434 --model my-model --out before.json
|
|
94
|
+
gguf-preflight check http://localhost:11434 --model my-model --out after.json
|
|
95
|
+
gguf-preflight diff before.json after.json --fail-on-regression
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Receipts keep only the file name of local GGUFs, not their folders.
|
|
99
|
+
|
|
100
|
+
## MCP server
|
|
101
|
+
|
|
102
|
+
The server exposes `preflight_check`, `inspect_gguf`, and `compare_receipts`. It is stricter than the CLI: endpoints must be local, and remote GGUFs must be `https` Hugging Face URLs.
|
|
103
|
+
|
|
104
|
+
**Claude Code**
|
|
105
|
+
|
|
106
|
+
```powershell
|
|
107
|
+
claude mcp add gguf-preflight -- uvx gguf-preflight mcp
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**Gemini CLI** (`~/.gemini/settings.json`)
|
|
111
|
+
|
|
112
|
+
```json
|
|
113
|
+
{
|
|
114
|
+
"mcpServers": {
|
|
115
|
+
"gguf-preflight": {
|
|
116
|
+
"command": "uvx",
|
|
117
|
+
"args": ["gguf-preflight", "mcp"]
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Python library
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from gguf_preflight.inspector.gguf_header import inspect_gguf
|
|
127
|
+
from gguf_preflight.runner import run_preflight
|
|
128
|
+
|
|
129
|
+
metadata = inspect_gguf("model.gguf")
|
|
130
|
+
receipt = run_preflight("http://localhost:11434", "qwen3:8b", agent="claude-code")
|
|
131
|
+
print(receipt.verdict, receipt.failure_codes)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Safety and limits
|
|
135
|
+
|
|
136
|
+
- **Model-written code runs on your machine.** The coding tasks execute the model's Python in a separate process with a timeout, a stripped environment, and an audit hook that blocks network, subprocess, and file-write calls. **This is not a security sandbox.** Only test models you are willing to run code from, or use a VM or container.
|
|
137
|
+
- **`tensor_sha256` is a layout fingerprint.** It hashes tensor names, shapes, and types, not the weights. Two fine-tunes with the same layout share it.
|
|
138
|
+
- **Determinism is best effort.** Requests use `temperature=0`, `top_p=1.0`, and `seed=42`. Output can still differ between runs, even on the same GPU, so `transcript_sha256` can change without a real regression. Treat a matching hash as strong evidence and a different hash as a prompt to read the diff.
|
|
139
|
+
- **Task scores are a smoke test.** They catch broken stacks. They do not rank models.
|
|
140
|
+
|
|
141
|
+
## What has been tested
|
|
142
|
+
|
|
143
|
+
Live runs used Windows 11, Ollama 0.40.12, an 8 GB RTX 5060, and an 8B Q4_K_M model:
|
|
144
|
+
|
|
145
|
+
| Setup | Verdict | Main finding | Full run |
|
|
146
|
+
|---|---|---|---|
|
|
147
|
+
| Ollama default, 4,096 context | `NO_GO` | A 3,933-token prompt was silently cut to 2,050 tokens | 126 to 194 s |
|
|
148
|
+
| Model maximum, 40,960 context, 69% on GPU | `CAUTION` | Below the 64k guidance for Claude Code, and slow | 404 to 428 s |
|
|
149
|
+
|
|
150
|
+
Run time depends mostly on generation speed, so slower or memory-constrained GPUs take several minutes.
|
|
151
|
+
|
|
152
|
+
**Not live-tested yet:** llama-server and LM Studio (covered by mocked HTTP tests only), Linux, and macOS.
|
|
153
|
+
|
|
154
|
+
## Development
|
|
155
|
+
|
|
156
|
+
```powershell
|
|
157
|
+
uv sync
|
|
158
|
+
uv run pytest
|
|
159
|
+
uv run mypy
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Task packs live in `suites/` as YAML, so new tasks need no probe code.
|
|
163
|
+
|
|
164
|
+
## License
|
|
165
|
+
|
|
166
|
+
MIT
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# gguf-preflight
|
|
2
|
+
|
|
3
|
+
Check a local GGUF model stack before you point an AI coding agent at it.
|
|
4
|
+
|
|
5
|
+
A local model can look healthy in a chat window and still break an agent. The server may silently cut off your context, drop tool calls, miss the prompt cache on every turn, or run a quant that writes fluent, broken code. `gguf-preflight` tests for those failures, gives a verdict, and prints the exact fix for your runtime.
|
|
6
|
+
|
|
7
|
+
- Works with **Ollama**, **llama-server**, **LM Studio**, and other local OpenAI- or Anthropic-compatible servers.
|
|
8
|
+
- Ships as a **CLI**, a **Python library**, and an **MCP server**.
|
|
9
|
+
- No account, no telemetry, no GPU kernel code. MIT licensed.
|
|
10
|
+
|
|
11
|
+
## Quick start
|
|
12
|
+
|
|
13
|
+
You need [uv](https://docs.astral.sh/uv/) and a running local server.
|
|
14
|
+
|
|
15
|
+
```powershell
|
|
16
|
+
uvx gguf-preflight check http://localhost:11434 --model qwen3:8b --agent claude-code
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Inspect any GGUF without downloading its weights:
|
|
20
|
+
|
|
21
|
+
```powershell
|
|
22
|
+
uvx gguf-preflight inspect https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/blob/main/tinyllama-1.1b-chat-v1.0.Q2_K.gguf
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## What it checks
|
|
26
|
+
|
|
27
|
+
| Layer | Check | Failure code |
|
|
28
|
+
|---|---|---|
|
|
29
|
+
| GGUF header | Missing tokenizer, pre-tokenizer, or chat template. Low-bit quant with no importance matrix | `METADATA_DEFECT` |
|
|
30
|
+
| Wire | Silent context truncation at 4k and 8k (16k with `--deep`) | `WIRE_BUG` |
|
|
31
|
+
| Wire | Tool calls in streaming and non-streaming mode: empty drops, plaintext leaks, invalid JSON | `WIRE_BUG` |
|
|
32
|
+
| Cache | Whether an identical prefix is reused on the next request | `PREFIX_CACHE_MISS` |
|
|
33
|
+
| Agent smoke | 10 coding tasks run against hidden tests | `QUANT_COLLAPSE` |
|
|
34
|
+
| Agent smoke | 8 strict tool tasks: right tool, types, arguments, forbidden tools | `QUANT_COLLAPSE` |
|
|
35
|
+
| Agent smoke | 4 multi-turn tasks: does a turn 1 rule survive to turn 4 | `QUANT_COLLAPSE` |
|
|
36
|
+
| Speed | TTFT, prompt tok/s, generation tok/s, cold vs warm prefill (median and IQR) | `LATENCY_BOTTLENECK` |
|
|
37
|
+
|
|
38
|
+
`QUANT_COLLAPSE` means the stack answers fluently but scores below a suite's failure threshold. That can be a damaged quant. It can also be a model that is simply too small for the tasks. Borderline scores are warnings and do not blame the quant.
|
|
39
|
+
|
|
40
|
+
## Verdicts and exit codes
|
|
41
|
+
|
|
42
|
+
| Verdict | Meaning | Exit code |
|
|
43
|
+
|---|---|---|
|
|
44
|
+
| `GO` | Nothing failed | 0 |
|
|
45
|
+
| `CAUTION` | Usable, with warnings | 0 |
|
|
46
|
+
| `NO_GO` | A check failed | 1 |
|
|
47
|
+
| `INCOMPLETE` | A core check could not run, for example the GPU ran out of memory | 2 |
|
|
48
|
+
|
|
49
|
+
## Commands
|
|
50
|
+
|
|
51
|
+
```text
|
|
52
|
+
gguf-preflight check <endpoint> --model <name> [--gguf <path-or-url>]
|
|
53
|
+
[--agent claude-code|cline|generic] [--json] [--out receipt.json]
|
|
54
|
+
[--deep] [--allow-remote]
|
|
55
|
+
gguf-preflight inspect <path-or-hf-url> [--json]
|
|
56
|
+
gguf-preflight diff <receipt_a.json> <receipt_b.json> [--json] [--fail-on-regression]
|
|
57
|
+
gguf-preflight mcp
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Public hosts are refused unless you pass `--allow-remote`. If your server needs a key, set `GGUF_PREFLIGHT_API_KEY`. It is never written to a receipt. Set `HF_TOKEN` for gated Hugging Face repos.
|
|
61
|
+
|
|
62
|
+
## Receipts
|
|
63
|
+
|
|
64
|
+
`--out` saves a JSON receipt with the verdict, GGUF metadata, runtime, hardware, every probe result, the fixes, and a hash of the test transcript. Compare two receipts after you change a quant, a flag, or a runtime version:
|
|
65
|
+
|
|
66
|
+
```powershell
|
|
67
|
+
gguf-preflight check http://localhost:11434 --model my-model --out before.json
|
|
68
|
+
gguf-preflight check http://localhost:11434 --model my-model --out after.json
|
|
69
|
+
gguf-preflight diff before.json after.json --fail-on-regression
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Receipts keep only the file name of local GGUFs, not their folders.
|
|
73
|
+
|
|
74
|
+
## MCP server
|
|
75
|
+
|
|
76
|
+
The server exposes `preflight_check`, `inspect_gguf`, and `compare_receipts`. It is stricter than the CLI: endpoints must be local, and remote GGUFs must be `https` Hugging Face URLs.
|
|
77
|
+
|
|
78
|
+
**Claude Code**
|
|
79
|
+
|
|
80
|
+
```powershell
|
|
81
|
+
claude mcp add gguf-preflight -- uvx gguf-preflight mcp
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
**Gemini CLI** (`~/.gemini/settings.json`)
|
|
85
|
+
|
|
86
|
+
```json
|
|
87
|
+
{
|
|
88
|
+
"mcpServers": {
|
|
89
|
+
"gguf-preflight": {
|
|
90
|
+
"command": "uvx",
|
|
91
|
+
"args": ["gguf-preflight", "mcp"]
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Python library
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from gguf_preflight.inspector.gguf_header import inspect_gguf
|
|
101
|
+
from gguf_preflight.runner import run_preflight
|
|
102
|
+
|
|
103
|
+
metadata = inspect_gguf("model.gguf")
|
|
104
|
+
receipt = run_preflight("http://localhost:11434", "qwen3:8b", agent="claude-code")
|
|
105
|
+
print(receipt.verdict, receipt.failure_codes)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Safety and limits
|
|
109
|
+
|
|
110
|
+
- **Model-written code runs on your machine.** The coding tasks execute the model's Python in a separate process with a timeout, a stripped environment, and an audit hook that blocks network, subprocess, and file-write calls. **This is not a security sandbox.** Only test models you are willing to run code from, or use a VM or container.
|
|
111
|
+
- **`tensor_sha256` is a layout fingerprint.** It hashes tensor names, shapes, and types, not the weights. Two fine-tunes with the same layout share it.
|
|
112
|
+
- **Determinism is best effort.** Requests use `temperature=0`, `top_p=1.0`, and `seed=42`. Output can still differ between runs, even on the same GPU, so `transcript_sha256` can change without a real regression. Treat a matching hash as strong evidence and a different hash as a prompt to read the diff.
|
|
113
|
+
- **Task scores are a smoke test.** They catch broken stacks. They do not rank models.
|
|
114
|
+
|
|
115
|
+
## What has been tested
|
|
116
|
+
|
|
117
|
+
Live runs used Windows 11, Ollama 0.40.12, an 8 GB RTX 5060, and an 8B Q4_K_M model:
|
|
118
|
+
|
|
119
|
+
| Setup | Verdict | Main finding | Full run |
|
|
120
|
+
|---|---|---|---|
|
|
121
|
+
| Ollama default, 4,096 context | `NO_GO` | A 3,933-token prompt was silently cut to 2,050 tokens | 126 to 194 s |
|
|
122
|
+
| Model maximum, 40,960 context, 69% on GPU | `CAUTION` | Below the 64k guidance for Claude Code, and slow | 404 to 428 s |
|
|
123
|
+
|
|
124
|
+
Run time depends mostly on generation speed, so slower or memory-constrained GPUs take several minutes.
|
|
125
|
+
|
|
126
|
+
**Not live-tested yet:** llama-server and LM Studio (covered by mocked HTTP tests only), Linux, and macOS.
|
|
127
|
+
|
|
128
|
+
## Development
|
|
129
|
+
|
|
130
|
+
```powershell
|
|
131
|
+
uv sync
|
|
132
|
+
uv run pytest
|
|
133
|
+
uv run mypy
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Task packs live in `suites/` as YAML, so new tasks need no probe code.
|
|
137
|
+
|
|
138
|
+
## License
|
|
139
|
+
|
|
140
|
+
MIT
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "gguf-preflight"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Deterministic preflight checks for local GGUF inference stacks before you connect an AI coding agent."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "sebas8866" }]
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
keywords = ["gguf", "llama.cpp", "ollama", "lm-studio", "mcp", "local-llm", "ai-agents"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 3 - Alpha",
|
|
13
|
+
"Environment :: Console",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
18
|
+
"Topic :: Software Development :: Testing",
|
|
19
|
+
"Typing :: Typed",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"fastmcp>=4.2.4",
|
|
23
|
+
"httpx>=0.28.1",
|
|
24
|
+
"pydantic>=2.0",
|
|
25
|
+
"pyyaml>=6.0.4",
|
|
26
|
+
"rich>=15.0.0",
|
|
27
|
+
"typer>=0.29.2",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Repository = "https://github.com/sebas8866/gguf-preflight"
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
gguf-preflight = "gguf_preflight.cli:main"
|
|
35
|
+
|
|
36
|
+
[build-system]
|
|
37
|
+
requires = ["hatchling>=1.27"]
|
|
38
|
+
build-backend = "hatchling.build"
|
|
39
|
+
|
|
40
|
+
[tool.hatch.build.targets.wheel]
|
|
41
|
+
packages = ["src/gguf_preflight"]
|
|
42
|
+
|
|
43
|
+
# suites/ lives at the repo root but must ship inside the wheel for uvx.
|
|
44
|
+
[tool.hatch.build.targets.wheel.force-include]
|
|
45
|
+
"suites" = "gguf_preflight/suites"
|
|
46
|
+
|
|
47
|
+
[tool.hatch.build.targets.sdist]
|
|
48
|
+
include = ["src", "suites", "tests", "README.md", "LICENSE", "pyproject.toml"]
|
|
49
|
+
|
|
50
|
+
[dependency-groups]
|
|
51
|
+
dev = [
|
|
52
|
+
"mypy>=2.4.1",
|
|
53
|
+
"pytest>=9.2.0",
|
|
54
|
+
"respx>=0.23.1",
|
|
55
|
+
"types-pyyaml>=6.0.12.20261122",
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
[tool.pytest.ini_options]
|
|
59
|
+
testpaths = ["tests"]
|
|
60
|
+
addopts = "-q"
|
|
61
|
+
|
|
62
|
+
[tool.mypy]
|
|
63
|
+
python_version = "3.12"
|
|
64
|
+
strict = true
|
|
65
|
+
files = ["src"]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Runtime adapters and detection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from gguf_preflight.adapters.base import EndpointError, RuntimeAdapter
|
|
8
|
+
from gguf_preflight.adapters.llama_server import LlamaServerAdapter
|
|
9
|
+
from gguf_preflight.adapters.lmstudio import LmStudioAdapter
|
|
10
|
+
from gguf_preflight.adapters.ollama import OllamaAdapter
|
|
11
|
+
|
|
12
|
+
ADAPTERS: tuple[type[RuntimeAdapter], ...] = (
|
|
13
|
+
OllamaAdapter,
|
|
14
|
+
LlamaServerAdapter,
|
|
15
|
+
LmStudioAdapter,
|
|
16
|
+
RuntimeAdapter,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def detect_adapter(http: httpx.Client, base_url: str) -> RuntimeAdapter:
|
|
21
|
+
"""Identify the runtime from JSON APIs, so other apps on a shared port are rejected."""
|
|
22
|
+
for adapter in ADAPTERS:
|
|
23
|
+
if adapter.matches(http, base_url):
|
|
24
|
+
return adapter(http, base_url)
|
|
25
|
+
raise EndpointError(
|
|
26
|
+
f"No inference server found at {base_url}. Expected Ollama, llama-server, "
|
|
27
|
+
"LM Studio, or an OpenAI-compatible /v1/models API."
|
|
28
|
+
)
|