localbenchmark 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- localbenchmark-0.1.0/LICENSE +22 -0
- localbenchmark-0.1.0/MANIFEST.in +2 -0
- localbenchmark-0.1.0/PKG-INFO +215 -0
- localbenchmark-0.1.0/README.md +183 -0
- localbenchmark-0.1.0/localbenchmark/__init__.py +39 -0
- localbenchmark-0.1.0/localbenchmark/examples.py +11 -0
- localbenchmark-0.1.0/localbenchmark/llm_plots.py +833 -0
- localbenchmark-0.1.0/localbenchmark/localbenchmark.py +1091 -0
- localbenchmark-0.1.0/localbenchmark/tests/test_localbenchmark.py +13 -0
- localbenchmark-0.1.0/localbenchmark.egg-info/PKG-INFO +215 -0
- localbenchmark-0.1.0/localbenchmark.egg-info/SOURCES.txt +15 -0
- localbenchmark-0.1.0/localbenchmark.egg-info/dependency_links.txt +1 -0
- localbenchmark-0.1.0/localbenchmark.egg-info/entry_points.txt +4 -0
- localbenchmark-0.1.0/localbenchmark.egg-info/requires.txt +10 -0
- localbenchmark-0.1.0/localbenchmark.egg-info/top_level.txt +1 -0
- localbenchmark-0.1.0/pyproject.toml +54 -0
- localbenchmark-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Erdogan Taskesen
|
|
4
|
+
localbenchmark - Python package
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: localbenchmark
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: localbenchmark: local LLM evaluation — coding benchmarks, long-context stress tests, and ranking plots via LM Studio.
|
|
5
|
+
Author-email: Erdogan Taskesen <erdogant@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://erdogant.github.io/localbenchmark
|
|
8
|
+
Project-URL: Download, https://github.com/erdogant/localbenchmark/archive/{version}.tar.gz
|
|
9
|
+
Keywords: Python,localbenchmark,LLM,benchmark,HumanEval,LM Studio,Machine Learning,AI
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Intended Audience :: Education
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: Unix
|
|
14
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
15
|
+
Classifier: Operating System :: MacOS
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: numpy>=2.0.0
|
|
22
|
+
Requires-Dist: pandas
|
|
23
|
+
Requires-Dist: matplotlib
|
|
24
|
+
Requires-Dist: requests
|
|
25
|
+
Requires-Dist: tqdm
|
|
26
|
+
Requires-Dist: openai
|
|
27
|
+
Requires-Dist: scikit-learn
|
|
28
|
+
Requires-Dist: pymupdf
|
|
29
|
+
Requires-Dist: json_repair
|
|
30
|
+
Requires-Dist: datasets
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
[](https://img.shields.io/pypi/pyversions/localbenchmark)
|
|
34
|
+
[](https://pypi.org/project/localbenchmark/)
|
|
35
|
+
[](https://erdogant.github.io/localbenchmark/)
|
|
36
|
+
[](https://github.com/erdogant/localbenchmark/)
|
|
37
|
+
[](https://pepy.tech/project/localbenchmark)
|
|
38
|
+
[](https://pepy.tech/project/localbenchmark)
|
|
39
|
+
[](https://github.com/erdogant/localbenchmark/blob/master/LICENSE)
|
|
40
|
+
[](https://github.com/erdogant/localbenchmark/network)
|
|
41
|
+
[](https://github.com/erdogant/localbenchmark/issues)
|
|
42
|
+
[](http://www.repostatus.org/#active)
|
|
43
|
+
[](https://erdogant.github.io/localbenchmark/pages/html/Documentation.html#medium-blog)
|
|
44
|
+
[](https://erdogant.github.io/localbenchmark/pages/html/Documentation.html#)
|
|
45
|
+
|
|
46
|
+
<div>
|
|
47
|
+
<a href="https://erdogant.github.io/localbenchmark/">
|
|
48
|
+
<img src="https://raw.githubusercontent.com/erdogant/localbenchmark/master/docs/figs/logo.png"
|
|
49
|
+
width="100"
|
|
50
|
+
align="left" />
|
|
51
|
+
</a>
|
|
52
|
+
|
|
53
|
+
**localbenchmark** is a Python package for **local LLM evaluation**: coding benchmarks, long-context stress tests, speed probes, and ranking plots — driven through [LM Studio](https://lmstudio.ai/) (OpenAI-compatible API).
|
|
54
|
+
It answers a practical question: *can this model still write correct code when the context window is large, and is it fast enough that you would actually use it?* ⭐️Star it if you like it⭐️
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
<br>
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
### Key Features
|
|
62
|
+
|
|
63
|
+
| Feature | Description |
|
|
64
|
+
|--------|-------------|
|
|
65
|
+
| **Coding benchmark** | HumanEval-style tasks via LM Studio; temperature 0 for comparable runs |
|
|
66
|
+
| **Long context** | Test fixed windows (e.g. 50k–180k); optional context *fill* so the KV cache is exercised |
|
|
67
|
+
| **Speed probe** | Prefill / decode tok/s at controlled tokens-in-cache levels |
|
|
68
|
+
| **Resume-safe CSVs** | Durable results; skip finished configs; retry LLM errors |
|
|
69
|
+
| **Slow-model abort** | Stop a config after a streak of low decode speed; ranking accounts for incomplete runs |
|
|
70
|
+
| **Rank-sum plots** | Pass rate, decode speed, TTFT, GPU peak, context — weighted ranking (model × context and per model) |
|
|
71
|
+
| **LLMlight helpers** | Lightweight local prompting utilities (optional local GGUF via llama-cpp) |
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
### Requirements
|
|
76
|
+
|
|
77
|
+
- Python **≥ 3.10**
|
|
78
|
+
- [LM Studio](https://lmstudio.ai/) running locally with a model loaded (default `http://localhost:1234`)
|
|
79
|
+
- Dependencies: `numpy`, `pandas`, `matplotlib`, `requests`, `tqdm`, `openai`, `scikit-learn`, `pymupdf`, `json_repair`, `datazets`
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
### Installation
|
|
84
|
+
|
|
85
|
+
##### Install from PyPI
|
|
86
|
+
```bash
|
|
87
|
+
pip install localbenchmark
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
##### Install from GitHub
|
|
91
|
+
```bash
|
|
92
|
+
pip install git+https://github.com/erdogant/localbenchmark
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
##### Install from source
|
|
96
|
+
```bash
|
|
97
|
+
git clone https://github.com/erdogant/localbenchmark
|
|
98
|
+
cd localbenchmark
|
|
99
|
+
pip install -r requirements.txt
|
|
100
|
+
pip install -e .
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
##### Check import
|
|
104
|
+
```python
|
|
105
|
+
import localbenchmark
|
|
106
|
+
print(localbenchmark.__version__)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
### Quick start — coding benchmark
|
|
112
|
+
|
|
113
|
+
Start LM Studio, enable the local server, load a coding model, then:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
# Full default suite (models × context windows)
|
|
117
|
+
python -m localbenchmark.llm_coding_benchmark
|
|
118
|
+
|
|
119
|
+
# Subset of models / contexts
|
|
120
|
+
python -m localbenchmark.llm_coding_benchmark \
|
|
121
|
+
--models openai/gpt-oss-20b \
|
|
122
|
+
--contexts 50000 80000 100000 \
|
|
123
|
+
--base-url http://localhost:1234
|
|
124
|
+
|
|
125
|
+
# Stress the window: fill 50% of context with unrelated code before each task
|
|
126
|
+
python -m localbenchmark.llm_coding_benchmark --fill 0.5
|
|
127
|
+
|
|
128
|
+
# Speed probe only (no HumanEval)
|
|
129
|
+
python -m localbenchmark.llm_coding_benchmark --probe
|
|
130
|
+
|
|
131
|
+
# Resume / plots only
|
|
132
|
+
python -m localbenchmark.llm_coding_benchmark --plots-only
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**Useful flags**
|
|
136
|
+
|
|
137
|
+
| Flag | Meaning |
|
|
138
|
+
|------|---------|
|
|
139
|
+
| `--models` | One or more LM Studio model keys |
|
|
140
|
+
| `--contexts` | Context sizes in tokens (default includes 50k, 80k, 100k, 120k, 150k, 180k) |
|
|
141
|
+
| `--fill` | Fraction of the window to fill with filler code (0–0.9) |
|
|
142
|
+
| `--limit` | Run only the first N HumanEval problems |
|
|
143
|
+
| `--results` | Results CSV path (default `coding_benchmark_results.csv`) |
|
|
144
|
+
| `--probe` | Run the speed probe instead of HumanEval |
|
|
145
|
+
| `--no-plots` / `--plots-only` | Skip plots, or only regenerate plots |
|
|
146
|
+
|
|
147
|
+
Results are written to CSV. Configurations already present in the **summary** CSV are skipped on the next run. Configurations that abort for low tok/s stay in the summary so they are not reloaded endlessly.
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
151
|
+
### Plots and ranking
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
python -m localbenchmark.plot_results \
|
|
155
|
+
--results coding_benchmark_results.csv \
|
|
156
|
+
--out plots \
|
|
157
|
+
--rank-agg sum
|
|
158
|
+
|
|
159
|
+
# Only large windows (context_actual > 100000)
|
|
160
|
+
python -m localbenchmark.plot_results --min-context 100000 --out plots
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
| Flag | Meaning |
|
|
164
|
+
|------|---------|
|
|
165
|
+
| `--rank-agg` | Combine per-metric ranks: `sum` (default), `mean`, or `median` |
|
|
166
|
+
| `--min-context` | Keep rows with `context_actual` **strictly greater** than this value |
|
|
167
|
+
|
|
168
|
+
**Ranking metrics** (lower combined score is better):
|
|
169
|
+
|
|
170
|
+
1. Pass rate (early aborts count unfinished work against the model)
|
|
171
|
+
2. Decode speed (percentage of best — large speed gaps stay large)
|
|
172
|
+
3. Time to first token
|
|
173
|
+
4. Peak GPU memory
|
|
174
|
+
5. Context size (percentage of best)
|
|
175
|
+
|
|
176
|
+
Pass rate and decode speed are **weighted ×2**. Figures include pass rate vs context, tok/s, TTFT, GPU memory, quality–speed trade-off, and rank-sum panels (model × context and per model).
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
### What is measured
|
|
181
|
+
|
|
182
|
+
| Signal | Why it matters |
|
|
183
|
+
|--------|----------------|
|
|
184
|
+
| **Pass rate** | Generated code must pass the unit tests |
|
|
185
|
+
| **Decode speed (tok/s)** | Inference has to be usable, not only correct |
|
|
186
|
+
| **TTFT** | Latency before the first useful token |
|
|
187
|
+
| **Peak GPU memory** | Whether the setup fits your hardware |
|
|
188
|
+
| **Context** | Behaviour as the window grows |
|
|
189
|
+
|
|
190
|
+
Default context grid:
|
|
191
|
+
|
|
192
|
+
```text
|
|
193
|
+
50k → 80k → 100k → 120k → 150k → 180k
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Any context sizes present in the results file are included in the rank plots (not limited to a hard-coded list).
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
### Resources and Links
|
|
201
|
+
|
|
202
|
+
- **Documentation:** [Website](https://erdogant.github.io/localbenchmark)
|
|
203
|
+
- **Blog:** [Medium / docs](https://erdogant.github.io/localbenchmark/pages/html/Documentation.html#medium-blog)
|
|
204
|
+
- **Issues:** [GitHub Issues](https://github.com/erdogant/localbenchmark/issues)
|
|
205
|
+
- **Source:** [github.com/erdogant/localbenchmark](https://github.com/erdogant/localbenchmark)
|
|
206
|
+
|
|
207
|
+
---
|
|
208
|
+
|
|
209
|
+
### Maintainer
|
|
210
|
+
|
|
211
|
+
* Erdogan Taskesen, GitHub: [erdogant](https://github.com/erdogant)
|
|
212
|
+
* Contributions are welcome.
|
|
213
|
+
* Yes! This library is entirely **free** but it runs on coffee! Feel free to support with a <a href="https://erdogant.github.io/donate/?currency=USD&amount=5">Coffee</a>.
|
|
214
|
+
|
|
215
|
+
[](https://www.buymeacoffee.com/erdogant)
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
[](https://img.shields.io/pypi/pyversions/localbenchmark)
|
|
2
|
+
[](https://pypi.org/project/localbenchmark/)
|
|
3
|
+
[](https://erdogant.github.io/localbenchmark/)
|
|
4
|
+
[](https://github.com/erdogant/localbenchmark/)
|
|
5
|
+
[](https://pepy.tech/project/localbenchmark)
|
|
6
|
+
[](https://pepy.tech/project/localbenchmark)
|
|
7
|
+
[](https://github.com/erdogant/localbenchmark/blob/master/LICENSE)
|
|
8
|
+
[](https://github.com/erdogant/localbenchmark/network)
|
|
9
|
+
[](https://github.com/erdogant/localbenchmark/issues)
|
|
10
|
+
[](http://www.repostatus.org/#active)
|
|
11
|
+
[](https://erdogant.github.io/localbenchmark/pages/html/Documentation.html#medium-blog)
|
|
12
|
+
[](https://erdogant.github.io/localbenchmark/pages/html/Documentation.html#)
|
|
13
|
+
|
|
14
|
+
<div>
|
|
15
|
+
<a href="https://erdogant.github.io/localbenchmark/">
|
|
16
|
+
<img src="https://raw.githubusercontent.com/erdogant/localbenchmark/master/docs/figs/logo.png"
|
|
17
|
+
width="100"
|
|
18
|
+
align="left" />
|
|
19
|
+
</a>
|
|
20
|
+
|
|
21
|
+
**localbenchmark** is a Python package for **local LLM evaluation**: coding benchmarks, long-context stress tests, speed probes, and ranking plots — driven through [LM Studio](https://lmstudio.ai/) (OpenAI-compatible API).
|
|
22
|
+
It answers a practical question: *can this model still write correct code when the context window is large, and is it fast enough that you would actually use it?* ⭐️Star it if you like it⭐️
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
<br>
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
### Key Features
|
|
30
|
+
|
|
31
|
+
| Feature | Description |
|
|
32
|
+
|--------|-------------|
|
|
33
|
+
| **Coding benchmark** | HumanEval-style tasks via LM Studio; temperature 0 for comparable runs |
|
|
34
|
+
| **Long context** | Test fixed windows (e.g. 50k–180k); optional context *fill* so the KV cache is exercised |
|
|
35
|
+
| **Speed probe** | Prefill / decode tok/s at controlled tokens-in-cache levels |
|
|
36
|
+
| **Resume-safe CSVs** | Durable results; skip finished configs; retry LLM errors |
|
|
37
|
+
| **Slow-model abort** | Stop a config after a streak of low decode speed; ranking accounts for incomplete runs |
|
|
38
|
+
| **Rank-sum plots** | Pass rate, decode speed, TTFT, GPU peak, context — weighted ranking (model × context and per model) |
|
|
39
|
+
| **LLMlight helpers** | Lightweight local prompting utilities (optional local GGUF via llama-cpp) |
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
### Requirements
|
|
44
|
+
|
|
45
|
+
- Python **≥ 3.10**
|
|
46
|
+
- [LM Studio](https://lmstudio.ai/) running locally with a model loaded (default `http://localhost:1234`)
|
|
47
|
+
- Dependencies: `numpy`, `pandas`, `matplotlib`, `requests`, `tqdm`, `openai`, `scikit-learn`, `pymupdf`, `json_repair`, `datazets`
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
### Installation
|
|
52
|
+
|
|
53
|
+
##### Install from PyPI
|
|
54
|
+
```bash
|
|
55
|
+
pip install localbenchmark
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
##### Install from GitHub
|
|
59
|
+
```bash
|
|
60
|
+
pip install git+https://github.com/erdogant/localbenchmark
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
##### Install from source
|
|
64
|
+
```bash
|
|
65
|
+
git clone https://github.com/erdogant/localbenchmark
|
|
66
|
+
cd localbenchmark
|
|
67
|
+
pip install -r requirements.txt
|
|
68
|
+
pip install -e .
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
##### Check import
|
|
72
|
+
```python
|
|
73
|
+
import localbenchmark
|
|
74
|
+
print(localbenchmark.__version__)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
### Quick start — coding benchmark
|
|
80
|
+
|
|
81
|
+
Start LM Studio, enable the local server, load a coding model, then:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
# Full default suite (models × context windows)
|
|
85
|
+
python -m localbenchmark.llm_coding_benchmark
|
|
86
|
+
|
|
87
|
+
# Subset of models / contexts
|
|
88
|
+
python -m localbenchmark.llm_coding_benchmark \
|
|
89
|
+
--models openai/gpt-oss-20b \
|
|
90
|
+
--contexts 50000 80000 100000 \
|
|
91
|
+
--base-url http://localhost:1234
|
|
92
|
+
|
|
93
|
+
# Stress the window: fill 50% of context with unrelated code before each task
|
|
94
|
+
python -m localbenchmark.llm_coding_benchmark --fill 0.5
|
|
95
|
+
|
|
96
|
+
# Speed probe only (no HumanEval)
|
|
97
|
+
python -m localbenchmark.llm_coding_benchmark --probe
|
|
98
|
+
|
|
99
|
+
# Resume / plots only
|
|
100
|
+
python -m localbenchmark.llm_coding_benchmark --plots-only
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
**Useful flags**
|
|
104
|
+
|
|
105
|
+
| Flag | Meaning |
|
|
106
|
+
|------|---------|
|
|
107
|
+
| `--models` | One or more LM Studio model keys |
|
|
108
|
+
| `--contexts` | Context sizes in tokens (default includes 50k, 80k, 100k, 120k, 150k, 180k) |
|
|
109
|
+
| `--fill` | Fraction of the window to fill with filler code (0–0.9) |
|
|
110
|
+
| `--limit` | Run only the first N HumanEval problems |
|
|
111
|
+
| `--results` | Results CSV path (default `coding_benchmark_results.csv`) |
|
|
112
|
+
| `--probe` | Run the speed probe instead of HumanEval |
|
|
113
|
+
| `--no-plots` / `--plots-only` | Skip plots, or only regenerate plots |
|
|
114
|
+
|
|
115
|
+
Results are written to CSV. Configurations already present in the **summary** CSV are skipped on the next run. Configurations that abort for low tok/s stay in the summary so they are not reloaded endlessly.
|
|
116
|
+
|
|
117
|
+
---
|
|
118
|
+
|
|
119
|
+
### Plots and ranking
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
python -m localbenchmark.plot_results \
|
|
123
|
+
--results coding_benchmark_results.csv \
|
|
124
|
+
--out plots \
|
|
125
|
+
--rank-agg sum
|
|
126
|
+
|
|
127
|
+
# Only large windows (context_actual > 100000)
|
|
128
|
+
python -m localbenchmark.plot_results --min-context 100000 --out plots
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
| Flag | Meaning |
|
|
132
|
+
|------|---------|
|
|
133
|
+
| `--rank-agg` | Combine per-metric ranks: `sum` (default), `mean`, or `median` |
|
|
134
|
+
| `--min-context` | Keep rows with `context_actual` **strictly greater** than this value |
|
|
135
|
+
|
|
136
|
+
**Ranking metrics** (lower combined score is better):
|
|
137
|
+
|
|
138
|
+
1. Pass rate (early aborts count unfinished work against the model)
|
|
139
|
+
2. Decode speed (percentage of best — large speed gaps stay large)
|
|
140
|
+
3. Time to first token
|
|
141
|
+
4. Peak GPU memory
|
|
142
|
+
5. Context size (percentage of best)
|
|
143
|
+
|
|
144
|
+
Pass rate and decode speed are **weighted ×2**. Figures include pass rate vs context, tok/s, TTFT, GPU memory, quality–speed trade-off, and rank-sum panels (model × context and per model).
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
### What is measured
|
|
149
|
+
|
|
150
|
+
| Signal | Why it matters |
|
|
151
|
+
|--------|----------------|
|
|
152
|
+
| **Pass rate** | Generated code must pass the unit tests |
|
|
153
|
+
| **Decode speed (tok/s)** | Inference has to be usable, not only correct |
|
|
154
|
+
| **TTFT** | Latency before the first useful token |
|
|
155
|
+
| **Peak GPU memory** | Whether the setup fits your hardware |
|
|
156
|
+
| **Context** | Behaviour as the window grows |
|
|
157
|
+
|
|
158
|
+
Default context grid:
|
|
159
|
+
|
|
160
|
+
```text
|
|
161
|
+
50k → 80k → 100k → 120k → 150k → 180k
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Any context sizes present in the results file are included in the rank plots (not limited to a hard-coded list).
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
### Resources and Links
|
|
169
|
+
|
|
170
|
+
- **Documentation:** [Website](https://erdogant.github.io/localbenchmark)
|
|
171
|
+
- **Blog:** [Medium / docs](https://erdogant.github.io/localbenchmark/pages/html/Documentation.html#medium-blog)
|
|
172
|
+
- **Issues:** [GitHub Issues](https://github.com/erdogant/localbenchmark/issues)
|
|
173
|
+
- **Source:** [github.com/erdogant/localbenchmark](https://github.com/erdogant/localbenchmark)
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
### Maintainer
|
|
178
|
+
|
|
179
|
+
* Erdogan Taskesen, GitHub: [erdogant](https://github.com/erdogant)
|
|
180
|
+
* Contributions are welcome.
|
|
181
|
+
* Yes! This library is entirely **free** but it runs on coffee! Feel free to support with a <a href="https://erdogant.github.io/donate/?currency=USD&amount=5">Coffee</a>.
|
|
182
|
+
|
|
183
|
+
[](https://www.buymeacoffee.com/erdogant)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import localbenchmark.localbenchmark as localbenchmark
|
|
3
|
+
import localbenchmark.llm_plots as llm_plots
|
|
4
|
+
|
|
5
|
+
__author__ = 'Erdogan Taskesen'
|
|
6
|
+
__email__ = 'erdogant@gmail.com'
|
|
7
|
+
__version__ = '0.1.0'
|
|
8
|
+
|
|
9
|
+
# Setup package-level logger
|
|
10
|
+
_logger = logging.getLogger('localbenchmark')
|
|
11
|
+
_log_handler = logging.StreamHandler()
|
|
12
|
+
_formatter = logging.Formatter(fmt='[{asctime}] [{name:<12.12}] [{levelname:<8}] {message}', style='{', datefmt='%d-%m-%Y %H:%M:%S')
|
|
13
|
+
_log_handler.setFormatter(_formatter)
|
|
14
|
+
_log_handler.setLevel(logging.DEBUG)
|
|
15
|
+
if not _logger.hasHandlers(): # avoid duplicate handlers if re-imported
|
|
16
|
+
_logger.addHandler(_log_handler)
|
|
17
|
+
_logger.setLevel(logging.DEBUG)
|
|
18
|
+
_logger.propagate = True # allow submodules to inherit this handler
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# module level doc-string
|
|
23
|
+
__doc__ = """
|
|
24
|
+
localbenchmark
|
|
25
|
+
=====================================================================
|
|
26
|
+
|
|
27
|
+
localbenchmark is for...
|
|
28
|
+
|
|
29
|
+
Example
|
|
30
|
+
-------
|
|
31
|
+
>>> from localbenchmark import localbenchmark
|
|
32
|
+
>>> model = localbenchmark()
|
|
33
|
+
>>> localbenchmark.plot()
|
|
34
|
+
|
|
35
|
+
References
|
|
36
|
+
----------
|
|
37
|
+
https://github.com/erdogant/localbenchmark
|
|
38
|
+
|
|
39
|
+
"""
|