depuzzle 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- depuzzle/__init__.py +3 -0
- depuzzle/backends/__init__.py +0 -0
- depuzzle/backends/base.py +14 -0
- depuzzle/backends/ollama.py +86 -0
- depuzzle/cli.py +232 -0
- depuzzle/exporters.py +23 -0
- depuzzle/loaders.py +88 -0
- depuzzle/mcp_server/__init__.py +0 -0
- depuzzle/mcp_server/server.py +12 -0
- depuzzle/mcp_server/tools.py +14 -0
- depuzzle/metrics.py +45 -0
- depuzzle/models.py +44 -0
- depuzzle/profiler.py +54 -0
- depuzzle-0.2.1.dist-info/METADATA +366 -0
- depuzzle-0.2.1.dist-info/RECORD +18 -0
- depuzzle-0.2.1.dist-info/WHEEL +4 -0
- depuzzle-0.2.1.dist-info/entry_points.txt +2 -0
- depuzzle-0.2.1.dist-info/licenses/LICENSE +21 -0
depuzzle/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import subprocess
|
|
3
|
+
|
|
4
|
+
import httpx
|
|
5
|
+
|
|
6
|
+
from depuzzle.backends.base import BaseBackend
|
|
7
|
+
from depuzzle.models import BackendInfo
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class OllamaBackend(BaseBackend):
|
|
11
|
+
|
|
12
|
+
def __init__(
|
|
13
|
+
self,
|
|
14
|
+
model: str,
|
|
15
|
+
host: str = "http://localhost:11434",
|
|
16
|
+
):
|
|
17
|
+
self.model = model
|
|
18
|
+
self.host = host.rstrip("/")
|
|
19
|
+
|
|
20
|
+
def _parse_ollama_ps(self, output: str) -> BackendInfo:
|
|
21
|
+
|
|
22
|
+
lines = output.strip().splitlines()
|
|
23
|
+
|
|
24
|
+
if len(lines) < 2:
|
|
25
|
+
raise RuntimeError("No running Ollama models found")
|
|
26
|
+
|
|
27
|
+
row = lines[1]
|
|
28
|
+
|
|
29
|
+
columns = row.split()
|
|
30
|
+
print(columns)
|
|
31
|
+
|
|
32
|
+
processor = f"{columns[4]} {columns[5]}"
|
|
33
|
+
context = int(columns[6])
|
|
34
|
+
|
|
35
|
+
return BackendInfo(
|
|
36
|
+
backend="ollama",
|
|
37
|
+
processor=processor,
|
|
38
|
+
context_length=context,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def get_info(self) -> BackendInfo:
|
|
42
|
+
result = subprocess.run(
|
|
43
|
+
["ollama", "ps"],
|
|
44
|
+
capture_output=True,
|
|
45
|
+
text=True,
|
|
46
|
+
check=True,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
output = result.stdout
|
|
50
|
+
|
|
51
|
+
return self._parse_ollama_ps(output)
|
|
52
|
+
|
|
53
|
+
def generate(self, prompt: str):
|
|
54
|
+
|
|
55
|
+
url = f"{self.host}/api/chat"
|
|
56
|
+
|
|
57
|
+
payload = {
|
|
58
|
+
"model": self.model,
|
|
59
|
+
"messages": [
|
|
60
|
+
{
|
|
61
|
+
"role": "user",
|
|
62
|
+
"content": prompt,
|
|
63
|
+
}
|
|
64
|
+
],
|
|
65
|
+
"stream": True,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
with httpx.stream(
|
|
69
|
+
"POST",
|
|
70
|
+
url,
|
|
71
|
+
json=payload,
|
|
72
|
+
timeout=None,
|
|
73
|
+
) as response:
|
|
74
|
+
|
|
75
|
+
if response.status_code != 200:
|
|
76
|
+
print(response.text)
|
|
77
|
+
|
|
78
|
+
response.raise_for_status()
|
|
79
|
+
|
|
80
|
+
for line in response.iter_lines():
|
|
81
|
+
|
|
82
|
+
if line:
|
|
83
|
+
data = json.loads(line)
|
|
84
|
+
|
|
85
|
+
if "message" in data:
|
|
86
|
+
yield data["message"]["content"]
|
depuzzle/cli.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import typer
|
|
2
|
+
from rich import box
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.table import Table
|
|
5
|
+
|
|
6
|
+
from depuzzle import __version__
|
|
7
|
+
from depuzzle.backends.ollama import OllamaBackend
|
|
8
|
+
from depuzzle.exporters import save_trace
|
|
9
|
+
from depuzzle.loaders import load_trace
|
|
10
|
+
from depuzzle.metrics import Metrics
|
|
11
|
+
from depuzzle.profiler import Profiler
|
|
12
|
+
|
|
13
|
+
console = Console()
|
|
14
|
+
|
|
15
|
+
app = typer.Typer(
|
|
16
|
+
name="depuzzle",
|
|
17
|
+
help="Inference profiler for local LLMs.",
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
profile_app = typer.Typer(help="Profile LLM inference requests.")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def format_change(key, value1, value2):
|
|
25
|
+
if not isinstance(value1, (int, float)):
|
|
26
|
+
return "-"
|
|
27
|
+
|
|
28
|
+
if value1 == 0:
|
|
29
|
+
return "-"
|
|
30
|
+
|
|
31
|
+
change = ((value2 - value1) / value1) * 100
|
|
32
|
+
|
|
33
|
+
if key in [
|
|
34
|
+
"latency_seconds",
|
|
35
|
+
"ttft_seconds",
|
|
36
|
+
]:
|
|
37
|
+
if change > 0:
|
|
38
|
+
return f"[red]{change:+.2f}% (slower)[/red]"
|
|
39
|
+
else:
|
|
40
|
+
return f"[green]{change:+.2f}% (faster)[/green]"
|
|
41
|
+
|
|
42
|
+
if key == "tokens_per_second":
|
|
43
|
+
if change > 0:
|
|
44
|
+
return f"[green]{change:+.2f}% (better)[/green]"
|
|
45
|
+
else:
|
|
46
|
+
return f"[red]{change:+.2f}% (worse)[/red]"
|
|
47
|
+
|
|
48
|
+
return f"{change:+.2f}%"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def add_metric_comparison(table, summary1, summary2):
|
|
52
|
+
|
|
53
|
+
for key in [
|
|
54
|
+
"model",
|
|
55
|
+
"tokens",
|
|
56
|
+
"latency_seconds",
|
|
57
|
+
"ttft_seconds",
|
|
58
|
+
"tokens_per_second",
|
|
59
|
+
]:
|
|
60
|
+
value1 = summary1[key]
|
|
61
|
+
value2 = summary2[key]
|
|
62
|
+
|
|
63
|
+
change = format_change(key, value1, value2)
|
|
64
|
+
|
|
65
|
+
table.add_row(
|
|
66
|
+
key,
|
|
67
|
+
str(value1),
|
|
68
|
+
str(value2),
|
|
69
|
+
change,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def add_runtime_comparison(table, summary1, summary2):
|
|
74
|
+
runtime1 = summary1.get("runtime")
|
|
75
|
+
runtime2 = summary2.get("runtime")
|
|
76
|
+
|
|
77
|
+
print(runtime1)
|
|
78
|
+
print(runtime2)
|
|
79
|
+
|
|
80
|
+
if not runtime1 and not runtime2:
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
table.add_section()
|
|
84
|
+
|
|
85
|
+
table.add_row(
|
|
86
|
+
"[bold]runtime[/bold]",
|
|
87
|
+
"",
|
|
88
|
+
"",
|
|
89
|
+
"",
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
for key in [
|
|
93
|
+
"backend",
|
|
94
|
+
"processor",
|
|
95
|
+
"context_length",
|
|
96
|
+
]:
|
|
97
|
+
value1 = runtime1.get(key, "N/A") if runtime1 else "N/A"
|
|
98
|
+
value2 = runtime2.get(key, "N/A") if runtime2 else "N/A"
|
|
99
|
+
|
|
100
|
+
if value1 == value2:
|
|
101
|
+
change = "same"
|
|
102
|
+
elif value1 == "N/A" or value2 == "N/A":
|
|
103
|
+
change = "N/A"
|
|
104
|
+
else:
|
|
105
|
+
change = "changed"
|
|
106
|
+
|
|
107
|
+
table.add_row(
|
|
108
|
+
f" {key}",
|
|
109
|
+
str(value1),
|
|
110
|
+
str(value2),
|
|
111
|
+
change,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def print_comparison(summary1, summary2):
|
|
116
|
+
table = Table(
|
|
117
|
+
title="Comparison",
|
|
118
|
+
box=box.SIMPLE_HEAVY,
|
|
119
|
+
show_header=True,
|
|
120
|
+
header_style="bold",
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
table.add_column("Metric", style="white", width=20)
|
|
124
|
+
table.add_column("Run 1", width=20)
|
|
125
|
+
table.add_column("Run 2", width=20)
|
|
126
|
+
table.add_column("Change", width=25)
|
|
127
|
+
|
|
128
|
+
add_metric_comparison(table, summary1, summary2)
|
|
129
|
+
add_runtime_comparison(table, summary1, summary2)
|
|
130
|
+
|
|
131
|
+
console.print(table)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def print_summary(trace):
|
|
135
|
+
"""
|
|
136
|
+
Print inference metrics.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
metrics = Metrics(trace)
|
|
140
|
+
|
|
141
|
+
summary = metrics.summary()
|
|
142
|
+
|
|
143
|
+
typer.echo("\n--- Trace Summary ---")
|
|
144
|
+
|
|
145
|
+
for key, value in summary.items():
|
|
146
|
+
if isinstance(value, dict):
|
|
147
|
+
typer.echo(f"{key}:")
|
|
148
|
+
for sub_key, sub_value in value.items():
|
|
149
|
+
typer.echo(f" {sub_key}: {sub_value}")
|
|
150
|
+
|
|
151
|
+
else:
|
|
152
|
+
typer.echo(f"{key}: {value}")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@profile_app.command()
|
|
156
|
+
def run(
|
|
157
|
+
model: str = typer.Option(
|
|
158
|
+
"qwen2.5:3b",
|
|
159
|
+
help="LLM model to use.",
|
|
160
|
+
),
|
|
161
|
+
prompt: str = typer.Option(
|
|
162
|
+
"Explain virtual memory in one paragraph.",
|
|
163
|
+
help="Prompt to send to the model.",
|
|
164
|
+
),
|
|
165
|
+
output: str = typer.Option(
|
|
166
|
+
None,
|
|
167
|
+
help="Save trace to JSON file.",
|
|
168
|
+
),
|
|
169
|
+
):
|
|
170
|
+
"""
|
|
171
|
+
Profile a single LLM inference request.
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
backend = OllamaBackend(model=model)
|
|
175
|
+
|
|
176
|
+
profiler = Profiler(backend)
|
|
177
|
+
|
|
178
|
+
trace = profiler.run(prompt)
|
|
179
|
+
|
|
180
|
+
print_summary(trace)
|
|
181
|
+
|
|
182
|
+
if output:
|
|
183
|
+
save_trace(trace, output)
|
|
184
|
+
|
|
185
|
+
typer.echo(f"\nTrace saved to {output}")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@app.command()
|
|
189
|
+
def version():
|
|
190
|
+
typer.echo(f"depuzzle {__version__}")
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
@app.command()
|
|
194
|
+
def compare(
|
|
195
|
+
run1: str,
|
|
196
|
+
run2: str,
|
|
197
|
+
):
|
|
198
|
+
"""
|
|
199
|
+
Compare two inference traces.
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
typer.echo(f"Comparing {run1}")
|
|
203
|
+
typer.echo(f"against {run2}")
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
trace1 = load_trace(run1)
|
|
207
|
+
trace2 = load_trace(run2)
|
|
208
|
+
|
|
209
|
+
except FileNotFoundError as e:
|
|
210
|
+
typer.echo(f"Error: {e}", err=True)
|
|
211
|
+
raise typer.Exit(code=1)
|
|
212
|
+
|
|
213
|
+
metrics1 = Metrics(trace1)
|
|
214
|
+
metrics2 = Metrics(trace2)
|
|
215
|
+
|
|
216
|
+
summary1 = metrics1.summary()
|
|
217
|
+
summary2 = metrics2.summary()
|
|
218
|
+
|
|
219
|
+
print_comparison(
|
|
220
|
+
summary1,
|
|
221
|
+
summary2,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
app.add_typer(
|
|
226
|
+
profile_app,
|
|
227
|
+
name="profile",
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
if __name__ == "__main__":
|
|
232
|
+
app()
|
depuzzle/exporters.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from dataclasses import asdict
|
|
3
|
+
|
|
4
|
+
from depuzzle.models import InferenceTrace
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def save_trace(
|
|
8
|
+
trace: InferenceTrace,
|
|
9
|
+
filename: str,
|
|
10
|
+
):
|
|
11
|
+
data = asdict(trace)
|
|
12
|
+
|
|
13
|
+
with open(
|
|
14
|
+
filename,
|
|
15
|
+
"w",
|
|
16
|
+
encoding="utf-8",
|
|
17
|
+
) as file:
|
|
18
|
+
json.dump(
|
|
19
|
+
data,
|
|
20
|
+
file,
|
|
21
|
+
indent=4,
|
|
22
|
+
default=str,
|
|
23
|
+
)
|
depuzzle/loaders.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from depuzzle.models import (
|
|
6
|
+
BackendInfo,
|
|
7
|
+
InferenceTrace,
|
|
8
|
+
TokenEvent,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def validate_trace_schema(data: dict, path: str):
|
|
13
|
+
required_fields = [
|
|
14
|
+
"model",
|
|
15
|
+
"prompt",
|
|
16
|
+
"start_time",
|
|
17
|
+
"end_time",
|
|
18
|
+
"tokens",
|
|
19
|
+
"total_latency",
|
|
20
|
+
"time_to_first_token",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
for field in required_fields:
|
|
24
|
+
if field not in data:
|
|
25
|
+
raise ValueError(f"Invalid trace format: missing field '{field}' in {path}")
|
|
26
|
+
|
|
27
|
+
if not isinstance(data["model"], str):
|
|
28
|
+
raise ValueError("Invalid trace format: model must be a string")
|
|
29
|
+
|
|
30
|
+
if not isinstance(data["prompt"], str):
|
|
31
|
+
raise ValueError("Invalid trace format: prompt must be a string")
|
|
32
|
+
|
|
33
|
+
if not isinstance(data["tokens"], list):
|
|
34
|
+
raise ValueError("Invalid trace format: tokens must be a list")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def load_trace(path: str | Path) -> InferenceTrace:
|
|
38
|
+
path = Path(path)
|
|
39
|
+
|
|
40
|
+
if not path.exists():
|
|
41
|
+
raise FileNotFoundError(f"Trace file not found: {path}")
|
|
42
|
+
|
|
43
|
+
if path.stat().st_size == 0:
|
|
44
|
+
raise ValueError(f"Trace file is empty: {path}")
|
|
45
|
+
|
|
46
|
+
try:
|
|
47
|
+
with path.open("r", encoding="utf-8") as f:
|
|
48
|
+
data = json.load(f)
|
|
49
|
+
|
|
50
|
+
except json.JSONDecodeError as e:
|
|
51
|
+
raise ValueError(f"Invalid JSON trace: {path}") from e
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
start_time = datetime.fromisoformat(data["start_time"])
|
|
55
|
+
end_time = datetime.fromisoformat(data["end_time"])
|
|
56
|
+
|
|
57
|
+
except ValueError as e:
|
|
58
|
+
raise ValueError("Invalid trace format: invalid timestamp") from e
|
|
59
|
+
|
|
60
|
+
backend_info = None
|
|
61
|
+
|
|
62
|
+
if data.get("backend_info"):
|
|
63
|
+
backend_info = BackendInfo(
|
|
64
|
+
backend=data["backend_info"]["backend"],
|
|
65
|
+
processor=data["backend_info"]["processor"],
|
|
66
|
+
context_length=data["backend_info"]["context_length"],
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
validate_trace_schema(data, str(path))
|
|
70
|
+
|
|
71
|
+
tokens = [
|
|
72
|
+
TokenEvent(
|
|
73
|
+
token=token["token"],
|
|
74
|
+
timestamp=datetime.fromisoformat(token["timestamp"]),
|
|
75
|
+
)
|
|
76
|
+
for token in data["tokens"]
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
return InferenceTrace(
|
|
80
|
+
model=data["model"],
|
|
81
|
+
prompt=data["prompt"],
|
|
82
|
+
start_time=start_time,
|
|
83
|
+
end_time=end_time,
|
|
84
|
+
tokens=tokens,
|
|
85
|
+
total_latency=data["total_latency"],
|
|
86
|
+
time_to_first_token=data["time_to_first_token"],
|
|
87
|
+
backend_info=backend_info,
|
|
88
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from depuzzle.backends.ollama import OllamaBackend
|
|
2
|
+
from depuzzle.metrics import Metrics
|
|
3
|
+
from depuzzle.profiler import Profiler
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def profile_model(model: str, prompt: str):
|
|
7
|
+
backend = OllamaBackend(model)
|
|
8
|
+
profiler = Profiler(backend)
|
|
9
|
+
|
|
10
|
+
trace = profiler.run(prompt)
|
|
11
|
+
|
|
12
|
+
metrics = Metrics(trace)
|
|
13
|
+
|
|
14
|
+
return metrics.summary()
|
depuzzle/metrics.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
class Metrics:
|
|
2
|
+
|
|
3
|
+
def __init__(self, trace):
|
|
4
|
+
self.trace = trace
|
|
5
|
+
|
|
6
|
+
def token_count(self):
|
|
7
|
+
return len(self.trace.tokens)
|
|
8
|
+
|
|
9
|
+
def latency_seconds(self):
|
|
10
|
+
return self.trace.total_latency
|
|
11
|
+
|
|
12
|
+
def ttft_seconds(self):
|
|
13
|
+
return self.trace.time_to_first_token
|
|
14
|
+
|
|
15
|
+
def tokens_per_second(self):
|
|
16
|
+
|
|
17
|
+
latency = self.latency_seconds()
|
|
18
|
+
|
|
19
|
+
if latency == 0:
|
|
20
|
+
return 0
|
|
21
|
+
|
|
22
|
+
return self.token_count() / latency
|
|
23
|
+
|
|
24
|
+
def runtime_details(self):
|
|
25
|
+
|
|
26
|
+
if self.trace.backend_info is not None:
|
|
27
|
+
return self.trace.backend_info
|
|
28
|
+
|
|
29
|
+
def summary(self):
|
|
30
|
+
summary = {
|
|
31
|
+
"model": self.trace.model,
|
|
32
|
+
"tokens": self.token_count(),
|
|
33
|
+
"latency_seconds": round(self.latency_seconds(), 3),
|
|
34
|
+
"ttft_seconds": round(self.ttft_seconds(), 3),
|
|
35
|
+
"tokens_per_second": round(self.tokens_per_second(), 2),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if self.runtime_details() is not None:
|
|
39
|
+
summary["runtime"] = {
|
|
40
|
+
"backend": self.runtime_details().backend,
|
|
41
|
+
"processor": self.runtime_details().processor,
|
|
42
|
+
"context_length": self.runtime_details().context_length,
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return summary
|
depuzzle/models.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class TokenEvent:
|
|
7
|
+
token: str
|
|
8
|
+
timestamp: datetime
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class BackendInfo:
|
|
13
|
+
backend: str
|
|
14
|
+
processor: str
|
|
15
|
+
context_length: int
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class InferenceTrace:
|
|
20
|
+
model: str
|
|
21
|
+
prompt: str
|
|
22
|
+
|
|
23
|
+
start_time: datetime
|
|
24
|
+
end_time: datetime
|
|
25
|
+
|
|
26
|
+
tokens: list[TokenEvent]
|
|
27
|
+
|
|
28
|
+
total_latency: float
|
|
29
|
+
time_to_first_token: float | None
|
|
30
|
+
|
|
31
|
+
backend_info: BackendInfo | None = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class InferenceResult:
|
|
36
|
+
model: str
|
|
37
|
+
prompt: str
|
|
38
|
+
|
|
39
|
+
token_count: int
|
|
40
|
+
|
|
41
|
+
total_latency: float
|
|
42
|
+
time_to_first_token: float | None
|
|
43
|
+
|
|
44
|
+
tokens_per_second: float
|
depuzzle/profiler.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from datetime import UTC, datetime
|
|
3
|
+
|
|
4
|
+
from depuzzle.models import (
|
|
5
|
+
InferenceTrace,
|
|
6
|
+
TokenEvent,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Profiler:
|
|
11
|
+
|
|
12
|
+
def __init__(self, backend):
|
|
13
|
+
self.backend = backend
|
|
14
|
+
|
|
15
|
+
def run(
|
|
16
|
+
self,
|
|
17
|
+
prompt: str,
|
|
18
|
+
) -> InferenceTrace:
|
|
19
|
+
|
|
20
|
+
events: list[TokenEvent] = []
|
|
21
|
+
|
|
22
|
+
start_time = datetime.now(UTC)
|
|
23
|
+
start_counter = time.perf_counter()
|
|
24
|
+
|
|
25
|
+
first_token_latency = None
|
|
26
|
+
|
|
27
|
+
for token in self.backend.generate(prompt):
|
|
28
|
+
|
|
29
|
+
current_time = datetime.now(UTC)
|
|
30
|
+
|
|
31
|
+
if first_token_latency is None:
|
|
32
|
+
first_token_latency = time.perf_counter() - start_counter
|
|
33
|
+
|
|
34
|
+
event = TokenEvent(
|
|
35
|
+
token=token,
|
|
36
|
+
timestamp=current_time,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
events.append(event)
|
|
40
|
+
|
|
41
|
+
end_time = datetime.now(UTC)
|
|
42
|
+
|
|
43
|
+
total_latency = time.perf_counter() - start_counter
|
|
44
|
+
|
|
45
|
+
return InferenceTrace(
|
|
46
|
+
model=self.backend.model,
|
|
47
|
+
prompt=prompt,
|
|
48
|
+
start_time=start_time,
|
|
49
|
+
end_time=end_time,
|
|
50
|
+
tokens=events,
|
|
51
|
+
total_latency=total_latency,
|
|
52
|
+
time_to_first_token=first_token_latency,
|
|
53
|
+
backend_info=self.backend.get_info(),
|
|
54
|
+
)
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: depuzzle
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Inference profiler for local LLMs
|
|
5
|
+
License-File: LICENSE
|
|
6
|
+
Requires-Python: <3.14,>=3.12
|
|
7
|
+
Requires-Dist: fastmcp
|
|
8
|
+
Requires-Dist: httpx
|
|
9
|
+
Requires-Dist: mcp
|
|
10
|
+
Requires-Dist: typer
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: black; extra == 'dev'
|
|
13
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
15
|
+
Requires-Dist: pytest-asyncio; extra == 'dev'
|
|
16
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# depuzzle
|
|
20
|
+
|
|
21
|
+
[](https://github.com/mananraheja/depuzzle/actions)
|
|
22
|
+
|
|
23
|
+
A lightweight Python tool for profiling, analyzing, and comparing LLM inference performance.
|
|
24
|
+
|
|
25
|
+
`depuzzle` helps developers profile local LLM inference runs, capture token-level traces, and compare performance across models and configurations.
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
### Profile an LLM inference run
|
|
32
|
+
|
|
33
|
+
depuzzle can profile local LLM inference requests and capture token-level timing information.
|
|
34
|
+
|
|
35
|
+
Example:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
depuzzle profile run \
|
|
39
|
+
--model qwen2.5:3b \
|
|
40
|
+
--prompt "Explain virtual memory in one paragraph." \
|
|
41
|
+
--output run1.json
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Example output:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
--- Trace Summary ---
|
|
48
|
+
|
|
49
|
+
model: qwen2.5:3b
|
|
50
|
+
tokens: 633
|
|
51
|
+
latency_seconds: 31.13s
|
|
52
|
+
ttft_seconds: 4.71s
|
|
53
|
+
tokens_per_second: 20.33
|
|
54
|
+
|
|
55
|
+
runtime:
|
|
56
|
+
backend: ollama
|
|
57
|
+
processor: 72%/28% CPU/GPU
|
|
58
|
+
context_length: 131072
|
|
59
|
+
|
|
60
|
+
Trace saved to run1.json
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Compare inference runs
|
|
64
|
+
|
|
65
|
+
depuzzle can compare two inference traces to evaluate model performance differences.
|
|
66
|
+
|
|
67
|
+
Example:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
depuzzle compare sample_traces/run1.json sample_traces/run2.json
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Example output:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
--- Comparison ---
|
|
77
|
+
|
|
78
|
+
Metric Run 1 Run 2 Change
|
|
79
|
+
------------------------------------------------------------------------------------------
|
|
80
|
+
model qwen2.5:3b llama3.2:3b -
|
|
81
|
+
tokens 633 555 -12.32%
|
|
82
|
+
latency_seconds 31.13 28.41 -8.74% (faster)
|
|
83
|
+
ttft_seconds 4.71 6.98 +48.07% (slower)
|
|
84
|
+
tokens_per_second 20.33 19.53 -3.93% (worse)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
runtime
|
|
88
|
+
backend ollama ollama same
|
|
89
|
+
processor 72%/28% CPU/GPU 100% CPU changed
|
|
90
|
+
context_length 131072 131072 same
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Features
|
|
96
|
+
|
|
97
|
+
- Profile local LLM inference runs
|
|
98
|
+
- Capture token-level inference traces
|
|
99
|
+
- Measure:
|
|
100
|
+
- Total latency
|
|
101
|
+
- Time to First Token (TTFT)
|
|
102
|
+
- Tokens per second
|
|
103
|
+
- Record runtime information
|
|
104
|
+
- Backend
|
|
105
|
+
- CPU/GPU processor split
|
|
106
|
+
- Context length
|
|
107
|
+
- Export traces as JSON
|
|
108
|
+
- Compare inference runs
|
|
109
|
+
- Automated testing with GitHub Actions
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## Supported Backends
|
|
114
|
+
|
|
115
|
+
Current:
|
|
116
|
+
|
|
117
|
+
- Ollama
|
|
118
|
+
|
|
119
|
+
Planned:
|
|
120
|
+
|
|
121
|
+
- MLX
|
|
122
|
+
- llama.cpp
|
|
123
|
+
- vLLM
|
|
124
|
+
- TensorRT-LLM
|
|
125
|
+
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
## Installation
|
|
129
|
+
|
|
130
|
+
Clone the repository:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
git clone https://github.com/mananraheja/depuzzle.git
|
|
134
|
+
cd depuzzle
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Create a virtual environment:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
python -m venv .venv
|
|
141
|
+
source .venv/bin/activate
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Install:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
pip install -e .
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
For development:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
pip install -e ".[dev]"
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
## Quick Start
|
|
159
|
+
|
|
160
|
+
Run profiler:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
depuzzle profile run \
|
|
164
|
+
--model <model_name> \
|
|
165
|
+
--prompt "<sample_prompt>" \
|
|
166
|
+
--output <path_to_output_file.json>
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Compare two inference runs:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
depuzzle compare run1.json run2.json
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Example:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
Metric Run 1 Run 2 Change
|
|
179
|
+
|
|
180
|
+
Total Latency 12.5s 10.2s -18%
|
|
181
|
+
Time To First Token 0.8s 0.6s -25%
|
|
182
|
+
Tokens Generated 256 256 -
|
|
183
|
+
Tokens/sec 20.4 25.1 +23%
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## Trace Format
|
|
189
|
+
|
|
190
|
+
`depuzzle` expects traces in JSON format:
|
|
191
|
+
```json
|
|
192
|
+
{
|
|
193
|
+
"model": "llama3.2:3b",
|
|
194
|
+
"prompt": "Hello world",
|
|
195
|
+
"start_time": "2026-07-22T10:00:00",
|
|
196
|
+
"end_time": "2026-07-22T10:00:12",
|
|
197
|
+
"tokens": [
|
|
198
|
+
{
|
|
199
|
+
"token": "Hello",
|
|
200
|
+
"timestamp": "2026-07-22T10:00:01"
|
|
201
|
+
}
|
|
202
|
+
],
|
|
203
|
+
"total_latency": 12.0,
|
|
204
|
+
"time_to_first_token": 1.0,
|
|
205
|
+
"backend_info": {
|
|
206
|
+
"backend": "ollama",
|
|
207
|
+
"processor": "72%/28% CPU/GPU",
|
|
208
|
+
"context_length": 131072
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## Architecture
|
|
216
|
+
|
|
217
|
+
`depuzzle` consists of two main workflows:
|
|
218
|
+
|
|
219
|
+
### 1. Profile Run
|
|
220
|
+
|
|
221
|
+
The profiling workflow executes an inference run and captures performance data.
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
User
|
|
225
|
+
|
|
|
226
|
+
v
|
|
227
|
+
depuzzle profile run
|
|
228
|
+
|
|
|
229
|
+
v
|
|
230
|
+
Profiler
|
|
231
|
+
|
|
|
232
|
+
v
|
|
233
|
+
Backend Adapter
|
|
234
|
+
|
|
|
235
|
+
v
|
|
236
|
+
Trace Recorder
|
|
237
|
+
|
|
|
238
|
+
v
|
|
239
|
+
trace.json
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
The generated trace contains:
|
|
244
|
+
- model information
|
|
245
|
+
- prompt metadata
|
|
246
|
+
- token generation timestamps
|
|
247
|
+
- latency measurements
|
|
248
|
+
- time-to-first-token
|
|
249
|
+
- backend information
|
|
250
|
+
- processor utilization
|
|
251
|
+
- context length
|
|
252
|
+
|
|
253
|
+
### 2. Analyze and Compare
|
|
254
|
+
|
|
255
|
+
The analysis workflow processes captured traces.
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
trace.json
|
|
259
|
+
|
|
|
260
|
+
v
|
|
261
|
+
Trace Loader
|
|
262
|
+
|
|
|
263
|
+
v
|
|
264
|
+
InferenceTrace Model
|
|
265
|
+
|
|
|
266
|
+
v
|
|
267
|
+
Metrics Engine
|
|
268
|
+
|
|
|
269
|
+
v
|
|
270
|
+
Comparison Report
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
Core components:
|
|
274
|
+
|
|
275
|
+
- **Profiler**: Captures inference execution events and generates trace files.
|
|
276
|
+
- **Loader**: Parses and validates trace JSON files.
|
|
277
|
+
- **Models**: Provides structured representations of inference runs.
|
|
278
|
+
- **Metrics**: Calculates performance statistics.
|
|
279
|
+
- **Compare**: Compares multiple inference runs.
|
|
280
|
+
|
|
281
|
+
---
|
|
282
|
+
|
|
283
|
+
## Development
|
|
284
|
+
|
|
285
|
+
Run tests:
|
|
286
|
+
|
|
287
|
+
```bash
|
|
288
|
+
pytest -m "not integration"
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Run integration tests (requires Ollama and downloaded local models):
|
|
292
|
+
```bash
|
|
293
|
+
pytest -m integration
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
Format code:
|
|
297
|
+
|
|
298
|
+
```bash
|
|
299
|
+
black .
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
Lint:
|
|
303
|
+
|
|
304
|
+
```bash
|
|
305
|
+
ruff check .
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Type checking:
|
|
309
|
+
|
|
310
|
+
```bash
|
|
311
|
+
mypy depuzzle
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
---
|
|
315
|
+
|
|
316
|
+
## Roadmap
|
|
317
|
+
|
|
318
|
+
### v0.1.0
|
|
319
|
+
- [x] Profile local LLM inference runs
|
|
320
|
+
- [x] Export traces to JSON
|
|
321
|
+
- [x] Calculate latency and throughput metrics
|
|
322
|
+
- [x] Compare inference runs
|
|
323
|
+
- [x] GitHub Actions CI
|
|
324
|
+
|
|
325
|
+
### v0.2.1
|
|
326
|
+
- [x] Backend runtime metadata
|
|
327
|
+
- [x] Runtime information in trace summaries
|
|
328
|
+
- [x] Runtime comparison between inference runs
|
|
329
|
+
|
|
330
|
+
### Future
|
|
331
|
+
|
|
332
|
+
#### Additional Backends
|
|
333
|
+
- [ ] MLX
|
|
334
|
+
- [ ] llama.cpp
|
|
335
|
+
- [ ] vLLM
|
|
336
|
+
- [ ] TensorRT-LLM
|
|
337
|
+
- [ ] Hugging Face Transformers
|
|
338
|
+
|
|
339
|
+
#### Profiling
|
|
340
|
+
- [ ] Memory usage
|
|
341
|
+
- [ ] GPU utilization
|
|
342
|
+
- [ ] Live inference profiling
|
|
343
|
+
- [ ] Batch benchmarking
|
|
344
|
+
- [ ] Multi-run statistical summaries
|
|
345
|
+
|
|
346
|
+
#### Visualization
|
|
347
|
+
- [ ] Interactive timeline visualization
|
|
348
|
+
- [ ] Trace diff visualization
|
|
349
|
+
- [ ] HTML report generation
|
|
350
|
+
|
|
351
|
+
#### AI Agent Integration
|
|
352
|
+
- [ ] MCP server
|
|
353
|
+
- [ ] Additional MCP tools
|
|
354
|
+
- [ ] Benchmark orchestration through MCP
|
|
355
|
+
|
|
356
|
+
#### Long-term
|
|
357
|
+
- [ ] Layer-by-layer profiling
|
|
358
|
+
- [ ] Distributed inference profiling
|
|
359
|
+
- [ ] Multi-node benchmarking
|
|
360
|
+
- [ ] Plugin system for custom backends
|
|
361
|
+
|
|
362
|
+
---
|
|
363
|
+
|
|
364
|
+
## LICENSE
|
|
365
|
+
|
|
366
|
+
MIT
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
depuzzle/__init__.py,sha256=3k5Qm4RDOL67qGKHBwF643GINu-o970aoGFa94j5w4E,75
|
|
2
|
+
depuzzle/cli.py,sha256=aDtCfScNZE416IzsiHVrGuy-NIKf5Urza9Ak_xj4sTc,4819
|
|
3
|
+
depuzzle/exporters.py,sha256=CIVCIi_QwzbYhIUie3ms76AjdNIEk42XNoDTad8MAc8,379
|
|
4
|
+
depuzzle/loaders.py,sha256=Fpxzu0yEu4qRKgabFQlzaObXTDbtxaVWTy32hYN8KSw,2413
|
|
5
|
+
depuzzle/metrics.py,sha256=G9RYheyTKGIcgQxqgigJHSr0mRazki1EoDZeIcYZbsk,1223
|
|
6
|
+
depuzzle/models.py,sha256=53R3kknp5sae7wW74uQzkVUaRzPIJSAkGMizo9ClTq4,663
|
|
7
|
+
depuzzle/profiler.py,sha256=OY6-RIgB9iW2_3JAyIaqX5yD7E6dkhsi36GVH-jJk5g,1243
|
|
8
|
+
depuzzle/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
depuzzle/backends/base.py,sha256=kz2sgudF_LV1100Ix5grs_yV9qelRE7fMze7EBm0xsU,242
|
|
10
|
+
depuzzle/backends/ollama.py,sha256=YYOIVm3ylreNDQ2pPq6BY_eKZqzyAU3SFtosCmfLrHc,1910
|
|
11
|
+
depuzzle/mcp_server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
depuzzle/mcp_server/server.py,sha256=PwahoqxHT93lCrNespm-nv7oug77Lsa8KW9ygcT0SHw,160
|
|
13
|
+
depuzzle/mcp_server/tools.py,sha256=TPuqNbKzKelO7bT-FOAje_j_zgFGJx3sbg7zOunIRZY,335
|
|
14
|
+
depuzzle-0.2.1.dist-info/METADATA,sha256=n8cPfdUKk9OVs8kox7Eu1Xvraup7lr8tGttxazybxGM,6682
|
|
15
|
+
depuzzle-0.2.1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
16
|
+
depuzzle-0.2.1.dist-info/entry_points.txt,sha256=fmwXFvxTo6mJwQqWk9s-4dJ_qwiMTOlMBIWSGGiJk6k,46
|
|
17
|
+
depuzzle-0.2.1.dist-info/licenses/LICENSE,sha256=I7uCETUb9W6Rp0wfvyxrFQ_gJnrl7Cv-kIwQo4hYgk0,1069
|
|
18
|
+
depuzzle-0.2.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Manan Raheja
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|