depuzzle 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. depuzzle-0.2.1/.github/workflows/ci.yml +55 -0
  2. depuzzle-0.2.1/.gitignore +5 -0
  3. depuzzle-0.2.1/.python-version +1 -0
  4. depuzzle-0.2.1/CHANGELOG.md +77 -0
  5. depuzzle-0.2.1/LICENSE +21 -0
  6. depuzzle-0.2.1/PKG-INFO +366 -0
  7. depuzzle-0.2.1/README.md +348 -0
  8. depuzzle-0.2.1/depuzzle/__init__.py +3 -0
  9. depuzzle-0.2.1/depuzzle/backends/__init__.py +0 -0
  10. depuzzle-0.2.1/depuzzle/backends/base.py +14 -0
  11. depuzzle-0.2.1/depuzzle/backends/ollama.py +86 -0
  12. depuzzle-0.2.1/depuzzle/cli.py +232 -0
  13. depuzzle-0.2.1/depuzzle/exporters.py +23 -0
  14. depuzzle-0.2.1/depuzzle/loaders.py +88 -0
  15. depuzzle-0.2.1/depuzzle/mcp_server/__init__.py +0 -0
  16. depuzzle-0.2.1/depuzzle/mcp_server/server.py +12 -0
  17. depuzzle-0.2.1/depuzzle/mcp_server/tools.py +14 -0
  18. depuzzle-0.2.1/depuzzle/metrics.py +45 -0
  19. depuzzle-0.2.1/depuzzle/models.py +44 -0
  20. depuzzle-0.2.1/depuzzle/profiler.py +54 -0
  21. depuzzle-0.2.1/examples/ollama_demo.py +18 -0
  22. depuzzle-0.2.1/examples/try_loaders.py +7 -0
  23. depuzzle-0.2.1/pyproject.toml +40 -0
  24. depuzzle-0.2.1/pytest.ini +3 -0
  25. depuzzle-0.2.1/run3.json +1979 -0
  26. depuzzle-0.2.1/sample_traces/cpu_cache_trace.json +1994 -0
  27. depuzzle-0.2.1/sample_traces/run1.json +2542 -0
  28. depuzzle-0.2.1/sample_traces/run2.json +2230 -0
  29. depuzzle-0.2.1/scripts/test_mcp_client.py +34 -0
  30. depuzzle-0.2.1/tests/conftest.py +59 -0
  31. depuzzle-0.2.1/tests/integration/test_ollama_backend.py +40 -0
  32. depuzzle-0.2.1/tests/test_backend_info.py +13 -0
  33. depuzzle-0.2.1/tests/test_compare_metrics.py +62 -0
  34. depuzzle-0.2.1/tests/test_loaders.py +117 -0
  35. depuzzle-0.2.1/tests/test_mcp_protocol.py +17 -0
  36. depuzzle-0.2.1/tests/test_mcp_server.py +13 -0
  37. depuzzle-0.2.1/tests/test_mcp_tools.py +13 -0
  38. depuzzle-0.2.1/tests/test_metrics.py +106 -0
  39. depuzzle-0.2.1/tests/test_models.py +60 -0
  40. depuzzle-0.2.1/tests/test_ollama_parser.py +18 -0
  41. depuzzle-0.2.1/tests/test_profiler.py +56 -0
@@ -0,0 +1,55 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - "**"
7
+ pull_request:
8
+ branches:
9
+ - main
10
+
11
+ jobs:
12
+ test:
13
+ runs-on: ubuntu-latest
14
+
15
+ steps:
16
+ - name: Checkout repository
17
+ uses: actions/checkout@v4
18
+
19
+ - name: Set up Python
20
+ uses: actions/setup-python@v5
21
+ with:
22
+ python-version: "3.12"
23
+
24
+ - name: Install project
25
+ run: |
26
+ python -m pip install -e .
27
+
28
+ - name: Install dependencies
29
+ run: |
30
+ python -m pip install --upgrade pip
31
+ pip install -e ".[dev]"
32
+
33
+ - name: Debug package
34
+ run: |
35
+ pwd
36
+ ls -la
37
+ ls -la depuzzle
38
+ python -c "import sys; print(sys.executable)"
39
+ python -c "import depuzzle; print(depuzzle.__file__)"
40
+
41
+ - name: Run tests
42
+ run: |
43
+ python -m pytest -m "not integration"
44
+
45
+ - name: Check formatting
46
+ run: |
47
+ python -m black --check .
48
+
49
+ - name: Run linter
50
+ run: |
51
+ python -m ruff check .
52
+
53
+ - name: Type check
54
+ run: |
55
+ python -m mypy depuzzle
@@ -0,0 +1,5 @@
1
+ .venv/
2
+ __pycache__/
3
+ .pytest_cache/
4
+ *.egg-info/
5
+ .DS_Store
@@ -0,0 +1 @@
1
+ 3.12.12
@@ -0,0 +1,77 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ ## [0.2.1] - 2026-08-12
6
+
7
+ ### Fixed
8
+
9
+ - TestPyPI package upload and successful clean install
10
+
11
+ ## [0.2.0] - 2026-07-28
12
+
13
+ ### Added
14
+
15
+ - Added runtime metadata collection for inference runs.
16
+ - Added backend runtime information to trace summaries:
17
+ - Backend name
18
+ - Processor utilization (CPU/GPU split)
19
+ - Context length
20
+ - Added runtime metadata to JSON trace outputs.
21
+ - Added runtime comparison support for `depuzzle compare`.
22
+ - Added shared test fixtures using `conftest.py`.
23
+ - Added integration test coverage for Ollama backend functionality.
24
+
25
+ ### Changed
26
+
27
+ - Updated trace summaries to display runtime information separately from inference metrics.
28
+ - Improved comparison output by separating runtime configuration from performance metrics.
29
+ - Refactored tests to use reusable fake backends and fake traces.
30
+ - Improved handling of missing runtime information.
31
+
32
+ ### Fixed
33
+
34
+ - Fixed trace summary output formatting.
35
+ - Fixed comparison handling when runtime metadata is unavailable.
36
+ - Fixed CI failures caused by integration tests requiring Ollama.
37
+
38
+ ### Testing
39
+
40
+ - Added unit tests for runtime metadata.
41
+ - Added tests for runtime comparison.
42
+ - Added Ollama integration test markers.
43
+ - Improved GitHub Actions CI workflow.
44
+
45
+ ## [0.1.0] - 2026-07-24
46
+
47
+ ### Added
48
+
49
+ - Initial release of depuzzle.
50
+ - CLI for profiling local LLM inference.
51
+ - Support for profiling Ollama models.
52
+ - Token-level inference trace collection.
53
+ - JSON trace export.
54
+ - Trace loading utilities.
55
+ - Inference metrics including:
56
+ - token count
57
+ - total latency
58
+ - time to first token (TTFT)
59
+ - tokens per second
60
+ - Trace comparison command.
61
+ - Rich terminal output for summaries and comparisons.
62
+ - Unit test suite.
63
+ - GitHub Actions continuous integration.
64
+ - Project documentation and installation instructions.
65
+
66
+ ### Changed
67
+
68
+ - Established the initial project architecture around:
69
+ - Backends
70
+ - Profiler
71
+ - Trace models
72
+ - Metrics
73
+ - CLI
74
+
75
+ ### Fixed
76
+
77
+ - Initial public release.
depuzzle-0.2.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Manan Raheja
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,366 @@
1
+ Metadata-Version: 2.5
2
+ Name: depuzzle
3
+ Version: 0.2.1
4
+ Summary: Inference profiler for local LLMs
5
+ License-File: LICENSE
6
+ Requires-Python: <3.14,>=3.12
7
+ Requires-Dist: fastmcp
8
+ Requires-Dist: httpx
9
+ Requires-Dist: mcp
10
+ Requires-Dist: typer
11
+ Provides-Extra: dev
12
+ Requires-Dist: black; extra == 'dev'
13
+ Requires-Dist: mypy; extra == 'dev'
14
+ Requires-Dist: pytest; extra == 'dev'
15
+ Requires-Dist: pytest-asyncio; extra == 'dev'
16
+ Requires-Dist: ruff; extra == 'dev'
17
+ Description-Content-Type: text/markdown
18
+
19
+ # depuzzle
20
+
21
+ [![CI](https://github.com/mananraheja/depuzzle/actions/workflows/ci.yml/badge.svg)](https://github.com/mananraheja/depuzzle/actions)
22
+
23
+ A lightweight Python tool for profiling, analyzing, and comparing LLM inference performance.
24
+
25
+ `depuzzle` helps developers profile local LLM inference runs, capture token-level traces, and compare performance across models and configurations.
26
+
27
+ ---
28
+
29
+ ## Usage
30
+
31
+ ### Profile an LLM inference run
32
+
33
+ depuzzle can profile local LLM inference requests and capture token-level timing information.
34
+
35
+ Example:
36
+
37
+ ```bash
38
+ depuzzle profile run \
39
+ --model qwen2.5:3b \
40
+ --prompt "Explain virtual memory in one paragraph." \
41
+ --output run1.json
42
+ ```
43
+
44
+ Example output:
45
+
46
+ ```bash
47
+ --- Trace Summary ---
48
+
49
+ model: qwen2.5:3b
50
+ tokens: 633
51
+ latency_seconds: 31.13s
52
+ ttft_seconds: 4.71s
53
+ tokens_per_second: 20.33
54
+
55
+ runtime:
56
+ backend: ollama
57
+ processor: 72%/28% CPU/GPU
58
+ context_length: 131072
59
+
60
+ Trace saved to run1.json
61
+ ```
62
+
63
+ ### Compare inference runs
64
+
65
+ depuzzle can compare two inference traces to evaluate model performance differences.
66
+
67
+ Example:
68
+
69
+ ```bash
70
+ depuzzle compare sample_traces/run1.json sample_traces/run2.json
71
+ ```
72
+
73
+ Example output:
74
+
75
+ ```bash
76
+ --- Comparison ---
77
+
78
+ Metric Run 1 Run 2 Change
79
+ ------------------------------------------------------------------------------------------
80
+ model qwen2.5:3b llama3.2:3b -
81
+ tokens 633 555 -12.32%
82
+ latency_seconds 31.13 28.41 -8.74% (faster)
83
+ ttft_seconds 4.71 6.98 +48.07% (slower)
84
+ tokens_per_second 20.33 19.53 -3.93% (worse)
85
+
86
+
87
+ runtime
88
+ backend ollama ollama same
89
+ processor 72%/28% CPU/GPU 100% CPU changed
90
+ context_length 131072 131072 same
91
+ ```
92
+
93
+ ---
94
+
95
+ ## Features
96
+
97
+ - Profile local LLM inference runs
98
+ - Capture token-level inference traces
99
+ - Measure:
100
+ - Total latency
101
+ - Time to First Token (TTFT)
102
+ - Tokens per second
103
+ - Record runtime information
104
+ - Backend
105
+ - CPU/GPU processor split
106
+ - Context length
107
+ - Export traces as JSON
108
+ - Compare inference runs
109
+ - Automated testing with GitHub Actions
110
+
111
+ ---
112
+
113
+ ## Supported Backends
114
+
115
+ Current:
116
+
117
+ - Ollama
118
+
119
+ Planned:
120
+
121
+ - MLX
122
+ - llama.cpp
123
+ - vLLM
124
+ - TensorRT-LLM
125
+
126
+ ---
127
+
128
+ ## Installation
129
+
130
+ Clone the repository:
131
+
132
+ ```bash
133
+ git clone https://github.com/mananraheja/depuzzle.git
134
+ cd depuzzle
135
+ ```
136
+
137
+ Create a virtual environment:
138
+
139
+ ```bash
140
+ python -m venv .venv
141
+ source .venv/bin/activate
142
+ ```
143
+
144
+ Install:
145
+
146
+ ```bash
147
+ pip install -e .
148
+ ```
149
+
150
+ For development:
151
+
152
+ ```bash
153
+ pip install -e ".[dev]"
154
+ ```
155
+
156
+ ---
157
+
158
+ ## Quick Start
159
+
160
+ Run profiler:
161
+
162
+ ```bash
163
+ depuzzle profile run \
164
+ --model <model_name> \
165
+ --prompt "<sample_prompt>" \
166
+ --output <path_to_output_file.json>
167
+ ```
168
+
169
+ Compare two inference runs:
170
+
171
+ ```bash
172
+ depuzzle compare run1.json run2.json
173
+ ```
174
+
175
+ Example:
176
+
177
+ ```bash
178
+ Metric Run 1 Run 2 Change
179
+
180
+ Total Latency 12.5s 10.2s -18%
181
+ Time To First Token 0.8s 0.6s -25%
182
+ Tokens Generated 256 256 -
183
+ Tokens/sec 20.4 25.1 +23%
184
+ ```
185
+
186
+ ---
187
+
188
+ ## Trace Format
189
+
190
+ `depuzzle` expects traces in JSON format:
191
+ ```json
192
+ {
193
+ "model": "llama3.2:3b",
194
+ "prompt": "Hello world",
195
+ "start_time": "2026-07-22T10:00:00",
196
+ "end_time": "2026-07-22T10:00:12",
197
+ "tokens": [
198
+ {
199
+ "token": "Hello",
200
+ "timestamp": "2026-07-22T10:00:01"
201
+ }
202
+ ],
203
+ "total_latency": 12.0,
204
+ "time_to_first_token": 1.0,
205
+ "backend_info": {
206
+ "backend": "ollama",
207
+ "processor": "72%/28% CPU/GPU",
208
+ "context_length": 131072
209
+ }
210
+ }
211
+ ```
212
+
213
+ ---
214
+
215
+ ## Architecture
216
+
217
+ `depuzzle` consists of two main workflows:
218
+
219
+ ### 1. Profile Run
220
+
221
+ The profiling workflow executes an inference run and captures performance data.
222
+
223
+ ```bash
224
+ User
225
+ |
226
+ v
227
+ depuzzle profile run
228
+ |
229
+ v
230
+ Profiler
231
+ |
232
+ v
233
+ Backend Adapter
234
+ |
235
+ v
236
+ Trace Recorder
237
+ |
238
+ v
239
+ trace.json
240
+ ```
241
+
242
+
243
+ The generated trace contains:
244
+ - model information
245
+ - prompt metadata
246
+ - token generation timestamps
247
+ - latency measurements
248
+ - time-to-first-token
249
+ - backend information
250
+ - processor utilization
251
+ - context length
252
+
253
+ ### 2. Analyze and Compare
254
+
255
+ The analysis workflow processes captured traces.
256
+
257
+ ```bash
258
+ trace.json
259
+ |
260
+ v
261
+ Trace Loader
262
+ |
263
+ v
264
+ InferenceTrace Model
265
+ |
266
+ v
267
+ Metrics Engine
268
+ |
269
+ v
270
+ Comparison Report
271
+ ```
272
+
273
+ Core components:
274
+
275
+ - **Profiler**: Captures inference execution events and generates trace files.
276
+ - **Loader**: Parses and validates trace JSON files.
277
+ - **Models**: Provides structured representations of inference runs.
278
+ - **Metrics**: Calculates performance statistics.
279
+ - **Compare**: Compares multiple inference runs.
280
+
281
+ ---
282
+
283
+ ## Development
284
+
285
+ Run tests:
286
+
287
+ ```bash
288
+ pytest -m "not integration"
289
+ ```
290
+
291
+ Run integration tests (requires Ollama and downloaded local models):
292
+ ```bash
293
+ pytest -m integration
294
+ ```
295
+
296
+ Format code:
297
+
298
+ ```bash
299
+ black .
300
+ ```
301
+
302
+ Lint:
303
+
304
+ ```bash
305
+ ruff check .
306
+ ```
307
+
308
+ Type checking:
309
+
310
+ ```bash
311
+ mypy depuzzle
312
+ ```
313
+
314
+ ---
315
+
316
+ ## Roadmap
317
+
318
+ ### v0.1.0
319
+ - [x] Profile local LLM inference runs
320
+ - [x] Export traces to JSON
321
+ - [x] Calculate latency and throughput metrics
322
+ - [x] Compare inference runs
323
+ - [x] GitHub Actions CI
324
+
325
+ ### v0.2.1
326
+ - [x] Backend runtime metadata
327
+ - [x] Runtime information in trace summaries
328
+ - [x] Runtime comparison between inference runs
329
+
330
+ ### Future
331
+
332
+ #### Additional Backends
333
+ - [ ] MLX
334
+ - [ ] llama.cpp
335
+ - [ ] vLLM
336
+ - [ ] TensorRT-LLM
337
+ - [ ] Hugging Face Transformers
338
+
339
+ #### Profiling
340
+ - [ ] Memory usage
341
+ - [ ] GPU utilization
342
+ - [ ] Live inference profiling
343
+ - [ ] Batch benchmarking
344
+ - [ ] Multi-run statistical summaries
345
+
346
+ #### Visualization
347
+ - [ ] Interactive timeline visualization
348
+ - [ ] Trace diff visualization
349
+ - [ ] HTML report generation
350
+
351
+ #### AI Agent Integration
352
+ - [ ] MCP server
353
+ - [ ] Additional MCP tools
354
+ - [ ] Benchmark orchestration through MCP
355
+
356
+ #### Long-term
357
+ - [ ] Layer-by-layer profiling
358
+ - [ ] Distributed inference profiling
359
+ - [ ] Multi-node benchmarking
360
+ - [ ] Plugin system for custom backends
361
+
362
+ ---
363
+
364
+ ## LICENSE
365
+
366
+ MIT