siftcode 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {siftcode-0.3.0 → siftcode-0.4.0}/PKG-INFO +191 -38
- {siftcode-0.3.0 → siftcode-0.4.0}/README.md +187 -36
- {siftcode-0.3.0 → siftcode-0.4.0}/pyproject.toml +12 -3
- siftcode-0.4.0/src/siftcode/__init__.py +5 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/_version.py +48 -4
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/cli.py +269 -51
- siftcode-0.4.0/src/siftcode/core/__init__.py +1 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/core/analyzer.py +219 -95
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/core/cache.py +80 -21
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/core/config.py +161 -21
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/core/llm.py +205 -16
- siftcode-0.4.0/src/siftcode/core/manual.py +412 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/core/prompts.py +33 -0
- siftcode-0.4.0/src/siftcode/data/hicolor/128x128/apps/siftcode-logo.png +0 -0
- siftcode-0.4.0/src/siftcode/data/hicolor/256x256/apps/siftcode-logo.png +0 -0
- siftcode-0.4.0/src/siftcode/data/hicolor/48x48/apps/siftcode-logo.png +0 -0
- siftcode-0.4.0/src/siftcode/gui/__init__.py +11 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/analyzer_view.py +12 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/app.py +54 -59
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/settings_view.py +59 -1
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode.egg-info/PKG-INFO +191 -38
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode.egg-info/SOURCES.txt +6 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode.egg-info/requires.txt +3 -1
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_analyzer.py +132 -6
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_cache.py +18 -2
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_cli.py +131 -7
- siftcode-0.4.0/tests/test_cli_extended.py +646 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_config.py +153 -3
- siftcode-0.4.0/tests/test_gui_extended.py +249 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_llm.py +206 -6
- siftcode-0.4.0/tests/test_manual.py +478 -0
- siftcode-0.3.0/src/siftcode/__init__.py +0 -3
- siftcode-0.3.0/src/siftcode/core/__init__.py +0 -1
- siftcode-0.3.0/src/siftcode/core/manual.py +0 -208
- siftcode-0.3.0/src/siftcode/gui/__init__.py +0 -9
- {siftcode-0.3.0 → siftcode-0.4.0}/LICENSE +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/setup.cfg +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/__main__.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/data/__init__.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/data/siftcode-logo.png +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/guide_window.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/project_list.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/widgets/__init__.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/widgets/exclusion_list.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/widgets/log_view.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/widgets/markdown_view.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/widgets/project_row.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode/gui/window.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode.egg-info/dependency_links.txt +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode.egg-info/entry_points.txt +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/src/siftcode.egg-info/top_level.txt +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_gui_analyzer.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_gui_project_list.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_gui_settings.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_gui_smoke.py +0 -0
- {siftcode-0.3.0 → siftcode-0.4.0}/tests/test_prompts.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: siftcode
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Local LLM-powered code analysis and developer guide generator
|
|
5
5
|
Author-email: Tuomas Lähteenmäki <lahtis@gmail.com>
|
|
6
6
|
License-Expression: GPL-3.0-or-later
|
|
@@ -14,14 +14,16 @@ Classifier: Environment :: X11 Applications :: GTK
|
|
|
14
14
|
Classifier: Intended Audience :: Developers
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
18
|
Classifier: Topic :: Software Development :: Documentation
|
|
18
19
|
Requires-Python: >=3.11
|
|
19
20
|
Description-Content-Type: text/markdown
|
|
20
21
|
License-File: LICENSE
|
|
21
22
|
Requires-Dist: requests>=2.31
|
|
22
|
-
Requires-Dist: PyGObject>=3.46
|
|
23
23
|
Requires-Dist: platformdirs>=4.0
|
|
24
24
|
Requires-Dist: tomli-w>=1.0
|
|
25
|
+
Provides-Extra: gui
|
|
26
|
+
Requires-Dist: PyGObject>=3.46; extra == "gui"
|
|
25
27
|
Provides-Extra: dev
|
|
26
28
|
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
27
29
|
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
@@ -71,11 +73,10 @@ generated in minutes.
|
|
|
71
73
|
- **Project-wide context**: file analyses see the root summary and
|
|
72
74
|
sibling files, producing coherent guides
|
|
73
75
|
- **Progress feedback**: a live progress bar and log during analysis
|
|
74
|
-
- **Cancellable**: stop an analysis mid-run without losing your work
|
|
75
76
|
- **Configurable per project**: model, context size, temperature, file
|
|
76
77
|
exclusions, and cache settings are all saved per project
|
|
77
78
|
- **Two front-ends**: a modern GTK4 + libadwaita GUI and a scriptable CLI
|
|
78
|
-
- **Tested**:
|
|
79
|
+
- **Tested**: 353 unit tests, 79 % overall coverage (88–100 % for core modules)
|
|
79
80
|
|
|
80
81
|
---
|
|
81
82
|
|
|
@@ -102,8 +103,8 @@ generated in minutes.
|
|
|
102
103
|
|
|
103
104
|
- **Linux** (X11 or Wayland)
|
|
104
105
|
- **Python 3.11** or newer
|
|
105
|
-
- **GTK 4.10+** and **libadwaita 1.
|
|
106
|
-
- **[Ollama](https://ollama.com)** running locally
|
|
106
|
+
- **GTK 4.10+** and **libadwaita 1.5+** (for the GUI)
|
|
107
|
+
- **[Ollama](https://ollama.com)** 0.1.24 or newer, running locally
|
|
107
108
|
- A code-capable LLM installed via Ollama, for example:
|
|
108
109
|
|
|
109
110
|
```bash
|
|
@@ -119,12 +120,20 @@ GPU and the model you choose.
|
|
|
119
120
|
|---|---|---|---|---|
|
|
120
121
|
| `qwen2.5-coder:7b` | ~5 GB | ~3 min | Basic | Small projects, laptops |
|
|
121
122
|
| **`qwen2.5-coder:14b`** | **~10 GB** | **~7 min** | **Good** | **Recommended default** |
|
|
122
|
-
| `qwen2.5-coder:32b` | ~22 GB | ~33 min | Excellent |
|
|
123
|
+
| `qwen2.5-coder:32b` | ~22 GB | ~33 min | Excellent | 24 GB+ GPUs, best quality |
|
|
124
|
+
|
|
125
|
+
**Quick rule of thumb by VRAM:**
|
|
126
|
+
|
|
127
|
+
- **< 8 GB** → use `7b`, or accept slow CPU-only runs
|
|
128
|
+
- **8–16 GB** → use `14b`; it's the sweet spot for most projects
|
|
129
|
+
- **24 GB+** → use `32b`. It fits comfortably (e.g. Quadro P6000,
|
|
130
|
+
RTX 3090, RTX 4090) and produces noticeably fewer hallucinations —
|
|
131
|
+
in particular it stops inventing files like `setup.py` in modern
|
|
132
|
+
`pyproject.toml` projects.
|
|
123
133
|
|
|
124
134
|
For CUDA GPUs, ensure your driver is recent enough for your Ollama
|
|
125
135
|
build. CPU-only operation works but is slow (expect 5–15× longer runs).
|
|
126
136
|
|
|
127
|
-
|
|
128
137
|
**Observations from real runs:**
|
|
129
138
|
|
|
130
139
|
- **14B** produces a comprehensive guide in ~7 minutes. It occasionally
|
|
@@ -135,45 +144,148 @@ build. CPU-only operation works but is slow (expect 5–15× longer runs).
|
|
|
135
144
|
concise and may skip smaller files.
|
|
136
145
|
- **7B** is useful for small projects or when you need a quick pass.
|
|
137
146
|
|
|
138
|
-
|
|
139
|
-
|
|
147
|
+
### Context length
|
|
148
|
+
|
|
149
|
+
The default `num_ctx` is **8192** tokens, which fits the default 14B
|
|
150
|
+
model on a 10 GB GPU comfortably.
|
|
151
|
+
|
|
152
|
+
**When to raise it:** The guide generation prompt includes the root
|
|
153
|
+
summary, all directory summaries, and all file summaries. For a
|
|
154
|
+
40-file project, this can be ~23,000 tokens. With `num_ctx=8192`,
|
|
155
|
+
Ollama truncates the context and the model may not see most of the
|
|
156
|
+
files — the guide ends up mentioning only a handful of modules.
|
|
157
|
+
|
|
158
|
+
If you have the VRAM, raise it per project:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
siftcode analyze myproject --num-ctx 32768
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Multi-model support
|
|
165
|
+
|
|
166
|
+
Different analysis stages have different needs. Per-file summaries
|
|
167
|
+
are simple tasks where a small model excels; the root summary and
|
|
168
|
+
the final guide require broader reasoning and benefit from a larger
|
|
169
|
+
model. Siftcode lets you assign a different model to each stage.
|
|
170
|
+
|
|
171
|
+
Stages: `root`, `file`, `directories`, `guide`.
|
|
172
|
+
|
|
173
|
+
**With a 24 GB GPU** (Quadro P6000, RTX 3090, RTX 4090), the best
|
|
174
|
+
configuration is:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
siftcode analyze myproject \
|
|
178
|
+
--model-stage root=qwen2.5-coder:32b \
|
|
179
|
+
--model-stage guide=qwen2.5-coder:32b
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
This takes ~9 minutes instead of ~33 (32B on every file) or
|
|
183
|
+
~7 (14B everywhere), and gives most of the quality benefit: the
|
|
184
|
+
32B model handles the two stages where its broader reasoning
|
|
185
|
+
matters most, while 14B handles the file summaries where speed
|
|
186
|
+
matters more than depth.
|
|
187
|
+
|
|
188
|
+
Persist the configuration so you don't have to type it every
|
|
189
|
+
time — the stage overrides are saved to the project's TOML file:
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
siftcode analyze myproject --model-stage guide=qwen2.5-coder:32b
|
|
193
|
+
siftcode show myproject
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
```text
|
|
197
|
+
Model:
|
|
198
|
+
Name: qwen2.5-coder:14b
|
|
199
|
+
Stages:
|
|
200
|
+
root (default)
|
|
201
|
+
file (default)
|
|
202
|
+
directories (default)
|
|
203
|
+
guide qwen2.5-coder:32b
|
|
204
|
+
Ollama URL: http://127.0.0.1:11434
|
|
205
|
+
num_ctx: 8192
|
|
206
|
+
spec_num_ctx: 16384
|
|
207
|
+
temperature: 0.0
|
|
208
|
+
timeout: 600s
|
|
209
|
+
|
|
210
|
+
```
|
|
211
|
+
You can also edit the TOML directly:
|
|
212
|
+
```toml
|
|
213
|
+
[model]
|
|
214
|
+
name = "qwen2.5-coder:14b"
|
|
215
|
+
|
|
216
|
+
[model.stages]
|
|
217
|
+
root = "qwen2.5-coder:32b"
|
|
218
|
+
guide = "qwen2.5-coder:32b"
|
|
219
|
+
```
|
|
220
|
+
How the cache behaves: the cache is keyed per model, so
|
|
221
|
+
switching a single stage's model does not invalidate the other
|
|
222
|
+
stages' cached results. If you later re-run with the same stage
|
|
223
|
+
models, the previous summaries are reused — you only pay for the
|
|
224
|
+
stages you actually changed.
|
|
225
|
+
|
|
226
|
+
Shortcut: to use the same model for every stage, use --model:
|
|
227
|
+
```bash
|
|
228
|
+
siftcode analyze myproject --model qwen2.5-coder:32b
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
**Tip — keeping both models loaded:** If you have enough VRAM to
|
|
232
|
+
hold both 14B and 32B simultaneously (~32 GB VRAM total), you can
|
|
233
|
+
avoid model reloads between stages:
|
|
140
234
|
|
|
141
235
|
```bash
|
|
142
236
|
export OLLAMA_MAX_LOADED_MODELS=2
|
|
143
237
|
export OLLAMA_KV_CACHE_TYPE=q8_0
|
|
144
238
|
```
|
|
145
239
|
|
|
146
|
-
|
|
147
|
-
guide without reloading models from disk.
|
|
240
|
+
---
|
|
148
241
|
|
|
149
|
-
|
|
150
|
-
build. CPU-only operation works but is slow (expect 5–15× longer runs).
|
|
242
|
+
### Advanced per-file specifications (`--advanced`)
|
|
151
243
|
|
|
152
|
-
|
|
244
|
+
The `--advanced` flag generates a detailed technical specification for
|
|
245
|
+
each source file instead of the usual developer guide. Specs are much
|
|
246
|
+
longer than regular file summaries, so they use a separate context
|
|
247
|
+
length: `spec_num_ctx` (default **16384**).
|
|
248
|
+
|
|
249
|
+
**On a 24 GB GPU**, 32B Q4_K_M plus a 16k KV cache is right at the
|
|
250
|
+
VRAM limit — Ollama offloads ~4% of the model to the CPU, which roughly
|
|
251
|
+
doubles the run time. Lowering the spec context to 12k keeps the whole
|
|
252
|
+
model on the GPU:
|
|
253
|
+
|
|
254
|
+
```bash
|
|
255
|
+
siftcode analyze myproject --advanced \
|
|
256
|
+
--model-stage file=qwen2.5-coder:32b \
|
|
257
|
+
--spec-num-ctx 12288
|
|
258
|
+
```
|
|
153
259
|
|
|
154
|
-
|
|
155
|
-
|
|
260
|
+
Verify with `ollama ps`: the `PROCESSOR` column should read `100% GPU`.
|
|
261
|
+
If it shows any CPU percentage, lower `--spec-num-ctx` further
|
|
262
|
+
(10240, 8192) until the model fits entirely on the GPU.
|
|
156
263
|
|
|
157
|
-
|
|
158
|
-
all directory summaries, and all file summaries. For a 40-file
|
|
159
|
-
project, this is roughly 23,000 tokens. With `num_ctx=8192`,
|
|
160
|
-
Ollama truncates the context and the model never sees most of
|
|
161
|
-
the files — the guide ends up mentioning only a handful of
|
|
162
|
-
modules.
|
|
264
|
+
`spec_num_ctx` is saved per project, so you only need the flag once:
|
|
163
265
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
266
|
+
```bash
|
|
267
|
+
siftcode analyze myproject --advanced \
|
|
268
|
+
--model-stage file=qwen2.5-coder:32b \
|
|
269
|
+
--spec-num-ctx 12288
|
|
270
|
+
siftcode show myproject
|
|
271
|
+
# → spec_num_ctx: 12288
|
|
272
|
+
```
|
|
168
273
|
|
|
169
|
-
**
|
|
170
|
-
|
|
274
|
+
> **Note:** `--advanced` mode with a 32B model is very slow (one
|
|
275
|
+
> document per file, each taking 10+ minutes on a 24 GB GPU). For
|
|
276
|
+
> projects with more than ~10 files, use the default 14B model:
|
|
277
|
+
>
|
|
278
|
+
> siftcode analyze myproject --advanced
|
|
171
279
|
|
|
172
280
|
---
|
|
173
281
|
|
|
174
282
|
## Installation
|
|
283
|
+
```bash
|
|
284
|
+
pip install siftcode
|
|
285
|
+
```
|
|
286
|
+
> Note: the GTK GUI requires system packages listed under System dependencies. The CLI works without them
|
|
175
287
|
|
|
176
|
-
###
|
|
288
|
+
### Development installation
|
|
177
289
|
|
|
178
290
|
```bash
|
|
179
291
|
git clone https://codeberg.org/lahtis/siftcode.git
|
|
@@ -187,8 +299,6 @@ pip install -e ".[dev]"
|
|
|
187
299
|
|
|
188
300
|
```bash
|
|
189
301
|
siftcode --version
|
|
190
|
-
# → siftcode 0.3.0
|
|
191
|
-
|
|
192
302
|
siftcode about
|
|
193
303
|
# → prints version, author, and license
|
|
194
304
|
```
|
|
@@ -244,6 +354,11 @@ siftcode analyze "My Project" --output ~/siftcode-output
|
|
|
244
354
|
# Analyze it with advanced per-file technical specifications
|
|
245
355
|
siftcode analyze "My Project" --output ~/siftcode-output --advanced
|
|
246
356
|
|
|
357
|
+
# Advanced + multi-model: 32B for specs, 14B for the rest
|
|
358
|
+
siftcode analyze "My Project" --advanced \
|
|
359
|
+
--model-stage file=qwen2.5-coder:32b \
|
|
360
|
+
--spec-num-ctx 12288
|
|
361
|
+
|
|
247
362
|
# Delete a project
|
|
248
363
|
siftcode delete "My Project"
|
|
249
364
|
```
|
|
@@ -268,11 +383,11 @@ siftcode cache gc --days 30 # prune entries unused for 30 days
|
|
|
268
383
|
|
|
269
384
|
### Output
|
|
270
385
|
|
|
271
|
-
Analysis produces
|
|
386
|
+
Analysis produces these files in the output directory:
|
|
272
387
|
|
|
273
|
-
- `
|
|
274
|
-
- `
|
|
275
|
-
- `
|
|
388
|
+
- `findings_<project>_<timestamp>.json` — raw structured findings (root, directories, files, errors)
|
|
389
|
+
- `guide_<project>_<timestamp>.md` — the generated developer guide
|
|
390
|
+
- `specs_<project>_<timestamp>/` — advanced per-file technical specifications (when `--advanced` is used)
|
|
276
391
|
|
|
277
392
|
---
|
|
278
393
|
|
|
@@ -304,6 +419,22 @@ human-readable and can be edited by hand if you prefer.
|
|
|
304
419
|
|
|
305
420
|
---
|
|
306
421
|
|
|
422
|
+
### Default exclusions
|
|
423
|
+
|
|
424
|
+
The default exclusion list already covers:
|
|
425
|
+
|
|
426
|
+
- Version control directories (`.git`, `.venv`, `node_modules`)
|
|
427
|
+
- Build artifacts (`build`, `dist`, `.pytest_cache`)
|
|
428
|
+
- Documentation directories (`docs`, `doc`)
|
|
429
|
+
- Binary file suffixes (`.png`, `.jpg`, `.zip`, `.so`)
|
|
430
|
+
- **Database files** (`.sqlite`, `.sqlite3`, `.db`, `.db3`, `.mdb`)
|
|
431
|
+
- Editor configs (`.editorconfig`, `.env`)
|
|
432
|
+
|
|
433
|
+
You can add or remove patterns per project from the GUI, or by editing
|
|
434
|
+
the `analysis.exclude` list in the project's TOML file.
|
|
435
|
+
|
|
436
|
+
---
|
|
437
|
+
|
|
307
438
|
## How it works
|
|
308
439
|
|
|
309
440
|
1. **Root analysis** — the LLM sees the project tree and infers the
|
|
@@ -315,13 +446,32 @@ human-readable and can be edited by hand if you prefer.
|
|
|
315
446
|
4. **Guide generation** — all of the above is composed into a
|
|
316
447
|
structured Markdown developer guide.
|
|
317
448
|
5. **Advanced specs (optional)** — when `--advanced` is used, individual
|
|
318
|
-
per-file technical specifications are generated and stored in `
|
|
449
|
+
per-file technical specifications are generated and stored in `specs_<project>_<timestamp>/`.
|
|
319
450
|
|
|
320
451
|
Each step is cached by SHA-256 hash of the input. When you re-run on an
|
|
321
452
|
unchanged project, only the guide is regenerated.
|
|
322
453
|
|
|
323
454
|
---
|
|
324
455
|
|
|
456
|
+
### Beyond documentation: spec-driven development
|
|
457
|
+
|
|
458
|
+
Because `--advanced` produces per-file technical specifications in Markdown
|
|
459
|
+
(`specs_<project>_<timestamp>/`), the output can be used as an implementation spec — not just
|
|
460
|
+
as documentation.
|
|
461
|
+
|
|
462
|
+
Feed `guide_<project>_<timestamp>.md` and the `specs_<project>_<timestamp>/` directory to an AI coding assistant and ask it to build the project from scratch. The result is a new codebase that follows the original architecture and interfaces. This makes Siftcode useful not only for understanding existing code, but also for:
|
|
463
|
+
|
|
464
|
+
- **Porting** a project to another language — the specs are language-agnostic
|
|
465
|
+
- **Rebuilding** a project on a clean foundation — same design, new implementation
|
|
466
|
+
- **Starting from a spec** — write or generate the specs first, then let the AI implement them
|
|
467
|
+
|
|
468
|
+
> **Note:** Specs are generated by an LLM and may contain inaccuracies
|
|
469
|
+
> (the 14B model occasionally invents files that don't exist). Review them
|
|
470
|
+
> before using them as an implementation spec. Larger models produce more
|
|
471
|
+
> accurate specs.
|
|
472
|
+
|
|
473
|
+
---
|
|
474
|
+
|
|
325
475
|
## Development
|
|
326
476
|
|
|
327
477
|
```bash
|
|
@@ -341,6 +491,9 @@ ruff check src/ tests/
|
|
|
341
491
|
# Type-check
|
|
342
492
|
mypy src/
|
|
343
493
|
```
|
|
494
|
+
> Note: GUI tests require a display. Use xvfb-run in headless
|
|
495
|
+
environments; they are skipped automatically when no display is
|
|
496
|
+
available.
|
|
344
497
|
|
|
345
498
|
### Project layout
|
|
346
499
|
|
|
@@ -355,7 +508,7 @@ siftcode/
|
|
|
355
508
|
│ │ ├── config.py # project configuration
|
|
356
509
|
│ │ ├── llm.py # Ollama client
|
|
357
510
|
│ │ ├── prompts.py # prompt templates
|
|
358
|
-
│ │ └── manual.py #
|
|
511
|
+
│ │ └── manual.py # advanced specification generator
|
|
359
512
|
│ └── gui/ # GTK4 front-end
|
|
360
513
|
│ ├── app.py
|
|
361
514
|
│ ├── window.py
|
|
@@ -40,11 +40,10 @@ generated in minutes.
|
|
|
40
40
|
- **Project-wide context**: file analyses see the root summary and
|
|
41
41
|
sibling files, producing coherent guides
|
|
42
42
|
- **Progress feedback**: a live progress bar and log during analysis
|
|
43
|
-
- **Cancellable**: stop an analysis mid-run without losing your work
|
|
44
43
|
- **Configurable per project**: model, context size, temperature, file
|
|
45
44
|
exclusions, and cache settings are all saved per project
|
|
46
45
|
- **Two front-ends**: a modern GTK4 + libadwaita GUI and a scriptable CLI
|
|
47
|
-
- **Tested**:
|
|
46
|
+
- **Tested**: 353 unit tests, 79 % overall coverage (88–100 % for core modules)
|
|
48
47
|
|
|
49
48
|
---
|
|
50
49
|
|
|
@@ -71,8 +70,8 @@ generated in minutes.
|
|
|
71
70
|
|
|
72
71
|
- **Linux** (X11 or Wayland)
|
|
73
72
|
- **Python 3.11** or newer
|
|
74
|
-
- **GTK 4.10+** and **libadwaita 1.
|
|
75
|
-
- **[Ollama](https://ollama.com)** running locally
|
|
73
|
+
- **GTK 4.10+** and **libadwaita 1.5+** (for the GUI)
|
|
74
|
+
- **[Ollama](https://ollama.com)** 0.1.24 or newer, running locally
|
|
76
75
|
- A code-capable LLM installed via Ollama, for example:
|
|
77
76
|
|
|
78
77
|
```bash
|
|
@@ -88,12 +87,20 @@ GPU and the model you choose.
|
|
|
88
87
|
|---|---|---|---|---|
|
|
89
88
|
| `qwen2.5-coder:7b` | ~5 GB | ~3 min | Basic | Small projects, laptops |
|
|
90
89
|
| **`qwen2.5-coder:14b`** | **~10 GB** | **~7 min** | **Good** | **Recommended default** |
|
|
91
|
-
| `qwen2.5-coder:32b` | ~22 GB | ~33 min | Excellent |
|
|
90
|
+
| `qwen2.5-coder:32b` | ~22 GB | ~33 min | Excellent | 24 GB+ GPUs, best quality |
|
|
91
|
+
|
|
92
|
+
**Quick rule of thumb by VRAM:**
|
|
93
|
+
|
|
94
|
+
- **< 8 GB** → use `7b`, or accept slow CPU-only runs
|
|
95
|
+
- **8–16 GB** → use `14b`; it's the sweet spot for most projects
|
|
96
|
+
- **24 GB+** → use `32b`. It fits comfortably (e.g. Quadro P6000,
|
|
97
|
+
RTX 3090, RTX 4090) and produces noticeably fewer hallucinations —
|
|
98
|
+
in particular it stops inventing files like `setup.py` in modern
|
|
99
|
+
`pyproject.toml` projects.
|
|
92
100
|
|
|
93
101
|
For CUDA GPUs, ensure your driver is recent enough for your Ollama
|
|
94
102
|
build. CPU-only operation works but is slow (expect 5–15× longer runs).
|
|
95
103
|
|
|
96
|
-
|
|
97
104
|
**Observations from real runs:**
|
|
98
105
|
|
|
99
106
|
- **14B** produces a comprehensive guide in ~7 minutes. It occasionally
|
|
@@ -104,45 +111,148 @@ build. CPU-only operation works but is slow (expect 5–15× longer runs).
|
|
|
104
111
|
concise and may skip smaller files.
|
|
105
112
|
- **7B** is useful for small projects or when you need a quick pass.
|
|
106
113
|
|
|
107
|
-
|
|
108
|
-
|
|
114
|
+
### Context length
|
|
115
|
+
|
|
116
|
+
The default `num_ctx` is **8192** tokens, which fits the default 14B
|
|
117
|
+
model on a 10 GB GPU comfortably.
|
|
118
|
+
|
|
119
|
+
**When to raise it:** The guide generation prompt includes the root
|
|
120
|
+
summary, all directory summaries, and all file summaries. For a
|
|
121
|
+
40-file project, this can be ~23,000 tokens. With `num_ctx=8192`,
|
|
122
|
+
Ollama truncates the context and the model may not see most of the
|
|
123
|
+
files — the guide ends up mentioning only a handful of modules.
|
|
124
|
+
|
|
125
|
+
If you have the VRAM, raise it per project:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
siftcode analyze myproject --num-ctx 32768
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Multi-model support
|
|
132
|
+
|
|
133
|
+
Different analysis stages have different needs. Per-file summaries
|
|
134
|
+
are simple tasks where a small model excels; the root summary and
|
|
135
|
+
the final guide require broader reasoning and benefit from a larger
|
|
136
|
+
model. Siftcode lets you assign a different model to each stage.
|
|
137
|
+
|
|
138
|
+
Stages: `root`, `file`, `directories`, `guide`.
|
|
139
|
+
|
|
140
|
+
**With a 24 GB GPU** (Quadro P6000, RTX 3090, RTX 4090), the best
|
|
141
|
+
configuration is:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
siftcode analyze myproject \
|
|
145
|
+
--model-stage root=qwen2.5-coder:32b \
|
|
146
|
+
--model-stage guide=qwen2.5-coder:32b
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
This takes ~9 minutes instead of ~33 (32B on every file) or
|
|
150
|
+
~7 (14B everywhere), and gives most of the quality benefit: the
|
|
151
|
+
32B model handles the two stages where its broader reasoning
|
|
152
|
+
matters most, while 14B handles the file summaries where speed
|
|
153
|
+
matters more than depth.
|
|
154
|
+
|
|
155
|
+
Persist the configuration so you don't have to type it every
|
|
156
|
+
time — the stage overrides are saved to the project's TOML file:
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
siftcode analyze myproject --model-stage guide=qwen2.5-coder:32b
|
|
160
|
+
siftcode show myproject
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
```text
|
|
164
|
+
Model:
|
|
165
|
+
Name: qwen2.5-coder:14b
|
|
166
|
+
Stages:
|
|
167
|
+
root (default)
|
|
168
|
+
file (default)
|
|
169
|
+
directories (default)
|
|
170
|
+
guide qwen2.5-coder:32b
|
|
171
|
+
Ollama URL: http://127.0.0.1:11434
|
|
172
|
+
num_ctx: 8192
|
|
173
|
+
spec_num_ctx: 16384
|
|
174
|
+
temperature: 0.0
|
|
175
|
+
timeout: 600s
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
You can also edit the TOML directly:
|
|
179
|
+
```toml
|
|
180
|
+
[model]
|
|
181
|
+
name = "qwen2.5-coder:14b"
|
|
182
|
+
|
|
183
|
+
[model.stages]
|
|
184
|
+
root = "qwen2.5-coder:32b"
|
|
185
|
+
guide = "qwen2.5-coder:32b"
|
|
186
|
+
```
|
|
187
|
+
How the cache behaves: the cache is keyed per model, so
|
|
188
|
+
switching a single stage's model does not invalidate the other
|
|
189
|
+
stages' cached results. If you later re-run with the same stage
|
|
190
|
+
models, the previous summaries are reused — you only pay for the
|
|
191
|
+
stages you actually changed.
|
|
192
|
+
|
|
193
|
+
Shortcut: to use the same model for every stage, use --model:
|
|
194
|
+
```bash
|
|
195
|
+
siftcode analyze myproject --model qwen2.5-coder:32b
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
**Tip — keeping both models loaded:** If you have enough VRAM to
|
|
199
|
+
hold both 14B and 32B simultaneously (~32 GB VRAM total), you can
|
|
200
|
+
avoid model reloads between stages:
|
|
109
201
|
|
|
110
202
|
```bash
|
|
111
203
|
export OLLAMA_MAX_LOADED_MODELS=2
|
|
112
204
|
export OLLAMA_KV_CACHE_TYPE=q8_0
|
|
113
205
|
```
|
|
114
206
|
|
|
115
|
-
|
|
116
|
-
guide without reloading models from disk.
|
|
207
|
+
---
|
|
117
208
|
|
|
118
|
-
|
|
119
|
-
build. CPU-only operation works but is slow (expect 5–15× longer runs).
|
|
209
|
+
### Advanced per-file specifications (`--advanced`)
|
|
120
210
|
|
|
121
|
-
|
|
211
|
+
The `--advanced` flag generates a detailed technical specification for
|
|
212
|
+
each source file instead of the usual developer guide. Specs are much
|
|
213
|
+
longer than regular file summaries, so they use a separate context
|
|
214
|
+
length: `spec_num_ctx` (default **16384**).
|
|
215
|
+
|
|
216
|
+
**On a 24 GB GPU**, 32B Q4_K_M plus a 16k KV cache is right at the
|
|
217
|
+
VRAM limit — Ollama offloads ~4% of the model to the CPU, which roughly
|
|
218
|
+
doubles the run time. Lowering the spec context to 12k keeps the whole
|
|
219
|
+
model on the GPU:
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
siftcode analyze myproject --advanced \
|
|
223
|
+
--model-stage file=qwen2.5-coder:32b \
|
|
224
|
+
--spec-num-ctx 12288
|
|
225
|
+
```
|
|
122
226
|
|
|
123
|
-
|
|
124
|
-
|
|
227
|
+
Verify with `ollama ps`: the `PROCESSOR` column should read `100% GPU`.
|
|
228
|
+
If it shows any CPU percentage, lower `--spec-num-ctx` further
|
|
229
|
+
(10240, 8192) until the model fits entirely on the GPU.
|
|
125
230
|
|
|
126
|
-
|
|
127
|
-
all directory summaries, and all file summaries. For a 40-file
|
|
128
|
-
project, this is roughly 23,000 tokens. With `num_ctx=8192`,
|
|
129
|
-
Ollama truncates the context and the model never sees most of
|
|
130
|
-
the files — the guide ends up mentioning only a handful of
|
|
131
|
-
modules.
|
|
231
|
+
`spec_num_ctx` is saved per project, so you only need the flag once:
|
|
132
232
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
233
|
+
```bash
|
|
234
|
+
siftcode analyze myproject --advanced \
|
|
235
|
+
--model-stage file=qwen2.5-coder:32b \
|
|
236
|
+
--spec-num-ctx 12288
|
|
237
|
+
siftcode show myproject
|
|
238
|
+
# → spec_num_ctx: 12288
|
|
239
|
+
```
|
|
137
240
|
|
|
138
|
-
**
|
|
139
|
-
|
|
241
|
+
> **Note:** `--advanced` mode with a 32B model is very slow (one
|
|
242
|
+
> document per file, each taking 10+ minutes on a 24 GB GPU). For
|
|
243
|
+
> projects with more than ~10 files, use the default 14B model:
|
|
244
|
+
>
|
|
245
|
+
> siftcode analyze myproject --advanced
|
|
140
246
|
|
|
141
247
|
---
|
|
142
248
|
|
|
143
249
|
## Installation
|
|
250
|
+
```bash
|
|
251
|
+
pip install siftcode
|
|
252
|
+
```
|
|
253
|
+
> Note: the GTK GUI requires system packages listed under System dependencies. The CLI works without them
|
|
144
254
|
|
|
145
|
-
###
|
|
255
|
+
### Development installation
|
|
146
256
|
|
|
147
257
|
```bash
|
|
148
258
|
git clone https://codeberg.org/lahtis/siftcode.git
|
|
@@ -156,8 +266,6 @@ pip install -e ".[dev]"
|
|
|
156
266
|
|
|
157
267
|
```bash
|
|
158
268
|
siftcode --version
|
|
159
|
-
# → siftcode 0.3.0
|
|
160
|
-
|
|
161
269
|
siftcode about
|
|
162
270
|
# → prints version, author, and license
|
|
163
271
|
```
|
|
@@ -213,6 +321,11 @@ siftcode analyze "My Project" --output ~/siftcode-output
|
|
|
213
321
|
# Analyze it with advanced per-file technical specifications
|
|
214
322
|
siftcode analyze "My Project" --output ~/siftcode-output --advanced
|
|
215
323
|
|
|
324
|
+
# Advanced + multi-model: 32B for specs, 14B for the rest
|
|
325
|
+
siftcode analyze "My Project" --advanced \
|
|
326
|
+
--model-stage file=qwen2.5-coder:32b \
|
|
327
|
+
--spec-num-ctx 12288
|
|
328
|
+
|
|
216
329
|
# Delete a project
|
|
217
330
|
siftcode delete "My Project"
|
|
218
331
|
```
|
|
@@ -237,11 +350,11 @@ siftcode cache gc --days 30 # prune entries unused for 30 days
|
|
|
237
350
|
|
|
238
351
|
### Output
|
|
239
352
|
|
|
240
|
-
Analysis produces
|
|
353
|
+
Analysis produces these files in the output directory:
|
|
241
354
|
|
|
242
|
-
- `
|
|
243
|
-
- `
|
|
244
|
-
- `
|
|
355
|
+
- `findings_<project>_<timestamp>.json` — raw structured findings (root, directories, files, errors)
|
|
356
|
+
- `guide_<project>_<timestamp>.md` — the generated developer guide
|
|
357
|
+
- `specs_<project>_<timestamp>/` — advanced per-file technical specifications (when `--advanced` is used)
|
|
245
358
|
|
|
246
359
|
---
|
|
247
360
|
|
|
@@ -273,6 +386,22 @@ human-readable and can be edited by hand if you prefer.
|
|
|
273
386
|
|
|
274
387
|
---
|
|
275
388
|
|
|
389
|
+
### Default exclusions
|
|
390
|
+
|
|
391
|
+
The default exclusion list already covers:
|
|
392
|
+
|
|
393
|
+
- Version control directories (`.git`, `.venv`, `node_modules`)
|
|
394
|
+
- Build artifacts (`build`, `dist`, `.pytest_cache`)
|
|
395
|
+
- Documentation directories (`docs`, `doc`)
|
|
396
|
+
- Binary file suffixes (`.png`, `.jpg`, `.zip`, `.so`)
|
|
397
|
+
- **Database files** (`.sqlite`, `.sqlite3`, `.db`, `.db3`, `.mdb`)
|
|
398
|
+
- Editor configs (`.editorconfig`, `.env`)
|
|
399
|
+
|
|
400
|
+
You can add or remove patterns per project from the GUI, or by editing
|
|
401
|
+
the `analysis.exclude` list in the project's TOML file.
|
|
402
|
+
|
|
403
|
+
---
|
|
404
|
+
|
|
276
405
|
## How it works
|
|
277
406
|
|
|
278
407
|
1. **Root analysis** — the LLM sees the project tree and infers the
|
|
@@ -284,13 +413,32 @@ human-readable and can be edited by hand if you prefer.
|
|
|
284
413
|
4. **Guide generation** — all of the above is composed into a
|
|
285
414
|
structured Markdown developer guide.
|
|
286
415
|
5. **Advanced specs (optional)** — when `--advanced` is used, individual
|
|
287
|
-
per-file technical specifications are generated and stored in `
|
|
416
|
+
per-file technical specifications are generated and stored in `specs_<project>_<timestamp>/`.
|
|
288
417
|
|
|
289
418
|
Each step is cached by SHA-256 hash of the input. When you re-run on an
|
|
290
419
|
unchanged project, only the guide is regenerated.
|
|
291
420
|
|
|
292
421
|
---
|
|
293
422
|
|
|
423
|
+
### Beyond documentation: spec-driven development
|
|
424
|
+
|
|
425
|
+
Because `--advanced` produces per-file technical specifications in Markdown
|
|
426
|
+
(`specs_<project>_<timestamp>/`), the output can be used as an implementation spec — not just
|
|
427
|
+
as documentation.
|
|
428
|
+
|
|
429
|
+
Feed `guide_<project>_<timestamp>.md` and the `specs_<project>_<timestamp>/` directory to an AI coding assistant and ask it to build the project from scratch. The result is a new codebase that follows the original architecture and interfaces. This makes Siftcode useful not only for understanding existing code, but also for:
|
|
430
|
+
|
|
431
|
+
- **Porting** a project to another language — the specs are language-agnostic
|
|
432
|
+
- **Rebuilding** a project on a clean foundation — same design, new implementation
|
|
433
|
+
- **Starting from a spec** — write or generate the specs first, then let the AI implement them
|
|
434
|
+
|
|
435
|
+
> **Note:** Specs are generated by an LLM and may contain inaccuracies
|
|
436
|
+
> (the 14B model occasionally invents files that don't exist). Review them
|
|
437
|
+
> before using them as an implementation spec. Larger models produce more
|
|
438
|
+
> accurate specs.
|
|
439
|
+
|
|
440
|
+
---
|
|
441
|
+
|
|
294
442
|
## Development
|
|
295
443
|
|
|
296
444
|
```bash
|
|
@@ -310,6 +458,9 @@ ruff check src/ tests/
|
|
|
310
458
|
# Type-check
|
|
311
459
|
mypy src/
|
|
312
460
|
```
|
|
461
|
+
> Note: GUI tests require a display. Use xvfb-run in headless
|
|
462
|
+
environments; they are skipped automatically when no display is
|
|
463
|
+
available.
|
|
313
464
|
|
|
314
465
|
### Project layout
|
|
315
466
|
|
|
@@ -324,7 +475,7 @@ siftcode/
|
|
|
324
475
|
│ │ ├── config.py # project configuration
|
|
325
476
|
│ │ ├── llm.py # Ollama client
|
|
326
477
|
│ │ ├── prompts.py # prompt templates
|
|
327
|
-
│ │ └── manual.py #
|
|
478
|
+
│ │ └── manual.py # advanced specification generator
|
|
328
479
|
│ └── gui/ # GTK4 front-end
|
|
329
480
|
│ ├── app.py
|
|
330
481
|
│ ├── window.py
|