lcode-cli 0.2.2__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/ISSUE_TEMPLATE/model_request.yml +5 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/CHANGELOG.md +27 -1
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/PKG-INFO +7 -4
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/README.md +6 -3
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/SECURITY.md +2 -0
- lcode_cli-0.3.1/docs/bench.md +172 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/configuration.md +2 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/how-it-works.md +2 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/index.md +1 -1
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/models.md +2 -2
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/quickstart.md +1 -1
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/usage.md +36 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/mkdocs.yml +1 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/__init__.py +1 -1
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/agent.py +23 -0
- lcode_cli-0.3.1/src/lcode/bench.py +1082 -0
- lcode_cli-0.3.1/src/lcode/checkpoints.py +449 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/cli.py +92 -2
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/config.py +1 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/ollama.py +4 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/repl.py +163 -1
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/tools.py +3 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/conftest.py +6 -0
- lcode_cli-0.3.1/tests/test_bench.py +272 -0
- lcode_cli-0.3.1/tests/test_checkpoints.py +226 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.editorconfig +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/CODEOWNERS +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/dependabot.yml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/workflows/ci.yml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/workflows/docs.yml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.github/workflows/release.yml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.gitignore +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/.pre-commit-config.yaml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/AGENTS.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/CODE_OF_CONDUCT.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/CONTRIBUTING.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/LICENSE +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/assets/demo.svg +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/assets/extra.css +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/assets/logo.svg +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/assets/social-preview.png +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/changelog.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/contributing.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/installation.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/docs/troubleshooting.md +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/install.sh +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/overrides/main.html +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/pyproject.toml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/scripts/record_demo.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/scripts/social-preview.html +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/__main__.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/catalog.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/hardware.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/limits.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/models.toml +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/permissions.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/render.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/sessions.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/src/lcode/web.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_agent.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_catalog.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_cli.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_config.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_context_picker.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_permissions.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_sessions.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_tools.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/tests/test_web.py +0 -0
- {lcode_cli-0.2.2 → lcode_cli-0.3.1}/uv.lock +0 -0
|
@@ -35,6 +35,11 @@ body:
|
|
|
35
35
|
description: Context size used, tokens/s, whether tool calls (reading, editing, running commands) worked reliably, and anything odd.
|
|
36
36
|
validations:
|
|
37
37
|
required: true
|
|
38
|
+
- type: textarea
|
|
39
|
+
id: bench
|
|
40
|
+
attributes:
|
|
41
|
+
label: Output of `lcode bench <model> --markdown`
|
|
42
|
+
description: Optional, but the most useful thing you can add. It takes a few minutes; see https://nasser1941.github.io/lcode/bench/
|
|
38
43
|
- type: textarea
|
|
39
44
|
id: doctor
|
|
40
45
|
attributes:
|
|
@@ -6,6 +6,30 @@ All notable changes to lcode are documented here. The format follows
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.3.1] - 2026-09-30
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `lcode bench` scores models on this machine with eight small coding tasks: fixing a bug, finding
|
|
14
|
+
code, writing a script from a spec, renaming across files, a one-line edit in a long file,
|
|
15
|
+
recovering from failing commands, fixing a function to match its spec and adding a feature across
|
|
16
|
+
files. Each task runs in a temporary folder and has an automatic check, some with hidden tests.
|
|
17
|
+
It reports tasks passed, time, generation and prompt speed, tool-call errors and memory use.
|
|
18
|
+
`lcode bench model-a model-b` compares models, `--json` saves the results (a documented, versioned
|
|
19
|
+
format) and `--markdown` prints a table for a model test report.
|
|
20
|
+
|
|
21
|
+
## [0.3.0] - 2026-09-30
|
|
22
|
+
|
|
23
|
+
### Added
|
|
24
|
+
|
|
25
|
+
- `/undo` takes back the file changes of the last request: lcode saves a checkpoint before the model
|
|
26
|
+
first changes files in a request, and restores edited and deleted files and removes new ones,
|
|
27
|
+
including changes made by shell commands. `/undo` again goes further back, `/rewind N` goes back
|
|
28
|
+
to before request N (optionally removing those requests from the conversation too), and
|
|
29
|
+
`/checkpoints` lists them. Checkpoints live in a separate git repository in lcode's state folder,
|
|
30
|
+
so your own repository is never touched, and work in folders that aren't git repositories.
|
|
31
|
+
Turn them off with `lcode config set checkpoints false`.
|
|
32
|
+
|
|
9
33
|
## [0.2.2] - 2026-09-30
|
|
10
34
|
|
|
11
35
|
### Added
|
|
@@ -116,7 +140,9 @@ First public release.
|
|
|
116
140
|
- `AGENTS.md` project instructions and `/init` to generate them.
|
|
117
141
|
- One-line installer for Linux and macOS.
|
|
118
142
|
|
|
119
|
-
[Unreleased]: https://github.com/nasser1941/lcode/compare/v0.
|
|
143
|
+
[Unreleased]: https://github.com/nasser1941/lcode/compare/v0.3.1...HEAD
|
|
144
|
+
[0.3.1]: https://github.com/nasser1941/lcode/compare/v0.3.0...v0.3.1
|
|
145
|
+
[0.3.0]: https://github.com/nasser1941/lcode/compare/v0.2.2...v0.3.0
|
|
120
146
|
[0.2.2]: https://github.com/nasser1941/lcode/compare/v0.2.1...v0.2.2
|
|
121
147
|
[0.2.1]: https://github.com/nasser1941/lcode/compare/v0.2.0...v0.2.1
|
|
122
148
|
[0.2.0]: https://github.com/nasser1941/lcode/compare/v0.1.2...v0.2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: lcode-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: A local-first terminal coding agent powered by open-weight models on your own GPU or Mac, via Ollama.
|
|
5
5
|
Project-URL: Homepage, https://nasser1941.github.io/lcode/
|
|
6
6
|
Project-URL: Documentation, https://nasser1941.github.io/lcode/
|
|
@@ -73,9 +73,12 @@ through [Ollama](https://ollama.com), and when it needs current information it c
|
|
|
73
73
|
plans multi-step tasks with a visible todo list.
|
|
74
74
|
- **Long context.** 256K tokens with the default model (1M with Nemotron). You choose the window.
|
|
75
75
|
- **Hardware-aware.** Detects your NVIDIA GPU or Apple Silicon Mac and picks the best model and the
|
|
76
|
-
largest context that fits: `lcode setup` does it in one step.
|
|
76
|
+
largest context that fits: `lcode setup` does it in one step. `lcode bench` compares models on
|
|
77
|
+
real coding tasks on your machine.
|
|
77
78
|
- **Safe by default.** Every edit is shown as a diff and every command that isn't read-only needs
|
|
78
79
|
your approval. `auto-edit` and `yolo` modes when you want speed.
|
|
80
|
+
- **Undo.** lcode saves a checkpoint before the model changes files; `/undo` takes back the last
|
|
81
|
+
request's edits, new files and shell-command changes, without touching your git history.
|
|
79
82
|
- **Bring your own model.** Eight curated open-weight models, all tested end to end, or any Ollama model
|
|
80
83
|
with tool calling.
|
|
81
84
|
- **Web search when needed.** Looks up the latest versions, docs and error messages with Ollama web
|
|
@@ -123,6 +126,8 @@ lcode
|
|
|
123
126
|
| `lcode -c` / `lcode --resume` | continue the last session here / pick a saved session from a list |
|
|
124
127
|
| `lcode --model qwen3.5-9b --context 128k` | pick a model and context window for this session |
|
|
125
128
|
| `lcode models` / `lcode doctor` | what fits this machine / check the installation |
|
|
129
|
+
| `lcode bench qwen3.6-35b qwen3.5-9b` | compare models on small coding tasks on this machine |
|
|
130
|
+
| `/undo`, `/rewind` | take back the last request's file changes, or go back further |
|
|
126
131
|
| `/rename`, `/resume` | name the current session, resume a saved one |
|
|
127
132
|
| `/model`, `/context`, `/compact`, `/help` | switch model, resize the context window (pick from a list), summarize, list commands |
|
|
128
133
|
|
|
@@ -167,8 +172,6 @@ fills up. Details: [How it works](https://nasser1941.github.io/lcode/how-it-work
|
|
|
167
172
|
|
|
168
173
|
See the pinned [Roadmap issue](https://github.com/nasser1941/lcode/issues/10). Next up:
|
|
169
174
|
|
|
170
|
-
- [`/undo` and automatic checkpoints](https://github.com/nasser1941/lcode/issues/4)
|
|
171
|
-
- [`lcode bench`: score models on your own hardware](https://github.com/nasser1941/lcode/issues/5)
|
|
172
175
|
- [MCP tool servers](https://github.com/nasser1941/lcode/issues/6)
|
|
173
176
|
- [Homebrew formula](https://github.com/nasser1941/lcode/issues/8) and an [optional sandbox for shell commands](https://github.com/nasser1941/lcode/issues/9)
|
|
174
177
|
- More models in the catalog, and test reports from your hardware ([report one](https://github.com/nasser1941/lcode/issues/new?template=model_request.yml))
|
|
@@ -40,9 +40,12 @@ through [Ollama](https://ollama.com), and when it needs current information it c
|
|
|
40
40
|
plans multi-step tasks with a visible todo list.
|
|
41
41
|
- **Long context.** 256K tokens with the default model (1M with Nemotron). You choose the window.
|
|
42
42
|
- **Hardware-aware.** Detects your NVIDIA GPU or Apple Silicon Mac and picks the best model and the
|
|
43
|
-
largest context that fits: `lcode setup` does it in one step.
|
|
43
|
+
largest context that fits: `lcode setup` does it in one step. `lcode bench` compares models on
|
|
44
|
+
real coding tasks on your machine.
|
|
44
45
|
- **Safe by default.** Every edit is shown as a diff and every command that isn't read-only needs
|
|
45
46
|
your approval. `auto-edit` and `yolo` modes when you want speed.
|
|
47
|
+
- **Undo.** lcode saves a checkpoint before the model changes files; `/undo` takes back the last
|
|
48
|
+
request's edits, new files and shell-command changes, without touching your git history.
|
|
46
49
|
- **Bring your own model.** Eight curated open-weight models, all tested end to end, or any Ollama model
|
|
47
50
|
with tool calling.
|
|
48
51
|
- **Web search when needed.** Looks up the latest versions, docs and error messages with Ollama web
|
|
@@ -90,6 +93,8 @@ lcode
|
|
|
90
93
|
| `lcode -c` / `lcode --resume` | continue the last session here / pick a saved session from a list |
|
|
91
94
|
| `lcode --model qwen3.5-9b --context 128k` | pick a model and context window for this session |
|
|
92
95
|
| `lcode models` / `lcode doctor` | what fits this machine / check the installation |
|
|
96
|
+
| `lcode bench qwen3.6-35b qwen3.5-9b` | compare models on small coding tasks on this machine |
|
|
97
|
+
| `/undo`, `/rewind` | take back the last request's file changes, or go back further |
|
|
93
98
|
| `/rename`, `/resume` | name the current session, resume a saved one |
|
|
94
99
|
| `/model`, `/context`, `/compact`, `/help` | switch model, resize the context window (pick from a list), summarize, list commands |
|
|
95
100
|
|
|
@@ -134,8 +139,6 @@ fills up. Details: [How it works](https://nasser1941.github.io/lcode/how-it-work
|
|
|
134
139
|
|
|
135
140
|
See the pinned [Roadmap issue](https://github.com/nasser1941/lcode/issues/10). Next up:
|
|
136
141
|
|
|
137
|
-
- [`/undo` and automatic checkpoints](https://github.com/nasser1941/lcode/issues/4)
|
|
138
|
-
- [`lcode bench`: score models on your own hardware](https://github.com/nasser1941/lcode/issues/5)
|
|
139
142
|
- [MCP tool servers](https://github.com/nasser1941/lcode/issues/6)
|
|
140
143
|
- [Homebrew formula](https://github.com/nasser1941/lcode/issues/8) and an [optional sandbox for shell commands](https://github.com/nasser1941/lcode/issues/9)
|
|
141
144
|
- More models in the catalog, and test reports from your hardware ([report one](https://github.com/nasser1941/lcode/issues/new?template=model_request.yml))
|
|
@@ -37,6 +37,8 @@ user, in your working directory**. It is not a sandbox. Keep in mind:
|
|
|
37
37
|
snippets from your code. Use `web = ask` to approve each search, or `web = off` / `--no-web` to
|
|
38
38
|
keep everything local.
|
|
39
39
|
- Sessions (including file contents the model read) are stored in `~/.local/state/lcode/sessions`.
|
|
40
|
+
Checkpoints for `/undo` keep copies of your project's files (except ignored ones) in
|
|
41
|
+
`~/.local/state/lcode/checkpoints` for 14 days; turn them off with `checkpoints = false`.
|
|
40
42
|
|
|
41
43
|
Reports about bypassing the permission prompts, the read-only allowlist, or data leaving the
|
|
42
44
|
machine unexpectedly are especially welcome.
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# Benchmarking models
|
|
2
|
+
|
|
3
|
+
`lcode bench` answers the most common question about local models: *which one works best for
|
|
4
|
+
coding on my machine?* It runs a fixed set of small coding tasks against one or more models and
|
|
5
|
+
reports how many they solved, how fast they ran and how much memory they used.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
lcode bench # the configured model
|
|
9
|
+
lcode bench qwen3.6-35b qwen3.5-9b # compare models
|
|
10
|
+
lcode bench --tasks fix-bug,rename # run some of the tasks
|
|
11
|
+
lcode bench --json results.json # also save machine-readable results
|
|
12
|
+
lcode bench --markdown # also print a table to paste into a test report
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
lcode bench · NVIDIA GeForce RTX 4080 Laptop GPU (12 GB VRAM), 31 GB RAM · Ollama 0.32.15
|
|
17
|
+
|
|
18
|
+
lcode-qwen3.6-35b · 32K context · reasoning on · 8 tasks
|
|
19
|
+
Loaded in 16s · 22.3 GB, 45% on the GPU · reads prompts at 855 tok/s
|
|
20
|
+
✓ fix-bug Fix a bug so the tests pass 22s the tests pass
|
|
21
|
+
✓ find-code Answer with file:line 9s next_wait at backoff.py:13
|
|
22
|
+
✓ write-script Write a script from a spec 13s correct output, also on data it hadn't seen
|
|
23
|
+
✓ rename Rename a function across files 26s its tests and extra checks pass
|
|
24
|
+
✓ precise-edit One-line edit in a long file 10s changed only service_087's timeout
|
|
25
|
+
✓ recover Recover from failing commands 30s used --source and dropped the invalid line
|
|
26
|
+
✓ spec-fix Fix a function to match its spec 55s follows the whole docstring
|
|
27
|
+
✓ feature Add a feature across files 71s priorities work, including old todos
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
At the end it prints a table comparing the models (see [results](#results-on-a-12-gb-laptop) below).
|
|
31
|
+
|
|
32
|
+
A model usually takes 4 to 12 minutes; each task stops after 5 minutes (`--timeout`).
|
|
33
|
+
|
|
34
|
+
## The tasks
|
|
35
|
+
|
|
36
|
+
Every task runs in a new temporary folder, with every permission granted (`yolo` mode) and web
|
|
37
|
+
access off, and ends with an automatic check. The folders are deleted afterwards unless you pass
|
|
38
|
+
`--keep`, which also keeps each task's transcript (`lcode-bench.log`).
|
|
39
|
+
|
|
40
|
+
| Task | The model has to | Passes when |
|
|
41
|
+
|---|---|---|
|
|
42
|
+
| `fix-bug` | Find and fix a bug so the failing unit tests pass | The tests pass and the test file is unchanged |
|
|
43
|
+
| `find-code` | Answer a question about a small codebase | It names the right function and cites its `file:line` |
|
|
44
|
+
| `write-script` | Write a script from a spec and run it | The script prints the right output, also for data it never saw |
|
|
45
|
+
| `rename` | Rename a function across several files and tests | The old name is gone and extra hidden tests pass |
|
|
46
|
+
| `precise-edit` | Change one value in a 900-line file of near-identical entries | Exactly that one line changed |
|
|
47
|
+
| `recover` | Run a command that fails twice (a removed option, a bad input line) and work around it | The report is correct and only the bad line was dropped |
|
|
48
|
+
| `spec-fix` | Fix a function from a one-line bug report so it does everything its docstring says | Hidden tests of every rule in the docstring pass, edge cases included |
|
|
49
|
+
| `feature` | Add priorities to a small todo app (a new option, sorting, a marker in the output) | Its tests pass, and a scripted session gives the exact expected output, including for todos saved before the change |
|
|
50
|
+
|
|
51
|
+
The first six tasks are everyday work that good models get right; the last two need careful
|
|
52
|
+
reading and are where smaller models tend to slip. The checks are deterministic and can't be passed
|
|
53
|
+
by hard-coding: for example, `write-script` reruns the script on new data, and `rename`, `spec-fix`
|
|
54
|
+
and `feature` are checked with tests the model never saw.
|
|
55
|
+
|
|
56
|
+
## What's measured
|
|
57
|
+
|
|
58
|
+
| | |
|
|
59
|
+
|---|---|
|
|
60
|
+
| **Passed** | Tasks solved within the time limit |
|
|
61
|
+
| **Time** | Total time for the tasks (loading the model is reported separately) |
|
|
62
|
+
| **Generation** | Output tokens per second, across all the model's responses |
|
|
63
|
+
| **Prompt reading** | Prompt tokens per second: how fast the model reads files and tool output. Measured per model with fresh 4,000-token prompts (the faster of two readings, since the first one after loading includes warm-up), because during the tasks Ollama serves most of each request from its prompt cache |
|
|
64
|
+
| **Tool-call errors** | Tool calls that failed (wrong arguments, editing without reading first, unmatched edits) or that the model wrote as invalid JSON |
|
|
65
|
+
| **Memory** | The model's memory use and how much of it is on the GPU, from Ollama |
|
|
66
|
+
|
|
67
|
+
Results depend on the context window, so `lcode bench` uses **32K** by default on every machine,
|
|
68
|
+
which makes results comparable. When comparing models, each one is unloaded before the next starts
|
|
69
|
+
so every model gets the whole GPU; lcode warns if other models are already loaded. Use `--context 128k` to measure a larger window, and `--no-think`
|
|
70
|
+
to measure a model without reasoning.
|
|
71
|
+
|
|
72
|
+
## Results on a 12 GB laptop
|
|
73
|
+
|
|
74
|
+
RTX 4080 Laptop GPU (12 GB) with 31 GB of RAM, Ollama 0.32.15, 32K context, reasoning on:
|
|
75
|
+
|
|
76
|
+
| | `qwen3.6-35b` | `qwen3.5-9b` | `qwen3.5-4b` |
|
|
77
|
+
|---|---|---|---|
|
|
78
|
+
| fix-bug | ✓ 22s | ✓ 16s | ✓ 11s |
|
|
79
|
+
| find-code | ✓ 9s | ✓ 6s | ✓ 5s |
|
|
80
|
+
| write-script | ✓ 13s | ✓ 44s | ✗ 18s |
|
|
81
|
+
| rename | ✓ 26s | ✓ 28s | ✓ 30s |
|
|
82
|
+
| precise-edit | ✓ 10s | ✓ 12s | ✓ 8s |
|
|
83
|
+
| recover | ✓ 30s | ✗ 29s | ✓ 24s |
|
|
84
|
+
| spec-fix | ✓ 55s | ✓ 4m 40s | ✗ 2m 57s |
|
|
85
|
+
| feature | ✓ 1m 11s | ✗ 5m 00s | ✓ 47s |
|
|
86
|
+
| **Passed** | **8/8** | **6/8** | **6/8** |
|
|
87
|
+
| Time | 3m 57s | 11m 55s | 5m 20s |
|
|
88
|
+
| Generation | 71.4 tok/s | 60.1 tok/s | 91.3 tok/s |
|
|
89
|
+
| Prompt reading | 855 tok/s | 3,507 tok/s | 5,186 tok/s |
|
|
90
|
+
| Tool-call errors | 0 | 9 | 1 |
|
|
91
|
+
| Memory | 22.3 GB, 45% on GPU | 6.6 GB, 100% on GPU | 4.2 GB, 100% on GPU |
|
|
92
|
+
|
|
93
|
+
The small models fit entirely on the GPU and read prompts much faster, but the larger mixture-of-
|
|
94
|
+
experts model solves more and needs fewer attempts. The small models' failures were real mistakes:
|
|
95
|
+
copying `read_file`'s line numbers into a data file, ignoring negative numbers, missing an edge case
|
|
96
|
+
of the spec, and running out of time.
|
|
97
|
+
|
|
98
|
+
Models sample their answers, so a task near the edge of a model's ability can pass in one run and
|
|
99
|
+
fail in the next (`qwen3.6-35b` scored 7/8 in an earlier run). Compare totals rather than single
|
|
100
|
+
tasks, and run twice before drawing conclusions.
|
|
101
|
+
|
|
102
|
+
## Sharing results
|
|
103
|
+
|
|
104
|
+
Results from other machines, especially Apple Silicon Macs and other GPUs, decide which models lcode
|
|
105
|
+
recommends. Run
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
lcode bench qwen3.6-35b --markdown
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
and paste the table into a
|
|
112
|
+
[model test report](https://github.com/nasser1941/lcode/issues/new?template=model_request.yml).
|
|
113
|
+
|
|
114
|
+
## JSON format
|
|
115
|
+
|
|
116
|
+
`--json FILE` writes one document per run. The schema is versioned: fields may be added, but
|
|
117
|
+
existing fields keep their meaning until `schema` changes.
|
|
118
|
+
|
|
119
|
+
```json
|
|
120
|
+
{
|
|
121
|
+
"schema": 1,
|
|
122
|
+
"lcode": "0.3.1",
|
|
123
|
+
"ollama": "0.32.15",
|
|
124
|
+
"date": "2026-09-30T19:09:09+02:00",
|
|
125
|
+
"hardware": {
|
|
126
|
+
"os": "linux", "cpu": "13th Gen Intel(R) Core(TM) i9-13980HX", "ram_gib": 30.96,
|
|
127
|
+
"gpu": "NVIDIA GeForce RTX 4080 Laptop GPU", "vram_gib": 11.99, "unified": false,
|
|
128
|
+
"description": "NVIDIA GeForce RTX 4080 Laptop GPU (12 GB VRAM), 31 GB RAM"
|
|
129
|
+
},
|
|
130
|
+
"tasks": [{"id": "fix-bug", "title": "Fix a bug so the tests pass"}, …],
|
|
131
|
+
"runs": [
|
|
132
|
+
{
|
|
133
|
+
"model": "qwen3.6-35b",
|
|
134
|
+
"ollama_model": "lcode-qwen3.6-35b",
|
|
135
|
+
"context": 32768,
|
|
136
|
+
"num_batch": null,
|
|
137
|
+
"think": true,
|
|
138
|
+
"load_seconds": 16.2,
|
|
139
|
+
"prompt_tps": 854.6,
|
|
140
|
+
"memory_gb": 22.3,
|
|
141
|
+
"gpu_percent": 45,
|
|
142
|
+
"passed": 8,
|
|
143
|
+
"total": 8,
|
|
144
|
+
"seconds": 237.3,
|
|
145
|
+
"generation_tps": 71.4,
|
|
146
|
+
"tool_errors": 0,
|
|
147
|
+
"usage": {"requests": 56, "prompt_tokens": 221522, "prompt_ns": 73377987000,
|
|
148
|
+
"output_tokens": 11490, "output_ns": 161017657999},
|
|
149
|
+
"tasks": [
|
|
150
|
+
{"id": "fix-bug", "passed": true, "seconds": 21.9, "detail": "the tests pass",
|
|
151
|
+
"steps": 5, "tool_errors": 0, "error": null},
|
|
152
|
+
…
|
|
153
|
+
],
|
|
154
|
+
"error": null,
|
|
155
|
+
"interrupted": false
|
|
156
|
+
}
|
|
157
|
+
]
|
|
158
|
+
}
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
| Field | Meaning |
|
|
162
|
+
|---|---|
|
|
163
|
+
| `runs[].model` | The name you gave (catalog key or Ollama tag); `ollama_model` is the model that ran |
|
|
164
|
+
| `runs[].context`, `num_batch`, `think` | The settings used. lcode lowers the batch size or context if the GPU runs out of memory, and reports what it ended up using |
|
|
165
|
+
| `runs[].memory_gb`, `gpu_percent` | From Ollama's `/api/ps`; `null` if unavailable |
|
|
166
|
+
| `runs[].generation_tps` | Output tokens per second across the tasks; `null` if Ollama reported no timings |
|
|
167
|
+
| `runs[].prompt_tps` | Prompt tokens per second for a fresh 4,000-token prompt (the faster of two readings); `null` if it couldn't be measured |
|
|
168
|
+
| `runs[].usage` | Totals over the tasks as Ollama reports them. `prompt_tokens` includes tokens served from the prompt cache |
|
|
169
|
+
| `runs[].tasks[].steps` | Model responses in the task |
|
|
170
|
+
| `runs[].tasks[].error` | Why the task stopped early (`timed out after 300s`, an Ollama error), otherwise `null` |
|
|
171
|
+
| `runs[].error` | Why the model didn't run at all (for example, not installed), otherwise `null` |
|
|
172
|
+
| `runs[].interrupted` | You pressed ++ctrl+c++; the results cover the tasks that finished |
|
|
@@ -25,6 +25,7 @@ lcode config path # print the file location
|
|
|
25
25
|
| `web` | `on` | Web search and page fetching: `on`, `ask` (before each search/website) or `off` |
|
|
26
26
|
| `search_backend` | `auto` | `auto`, `ollama`, `brave`, `tavily` or `searxng`; `auto` uses the first one configured |
|
|
27
27
|
| `searxng_url` | | Your SearXNG instance, e.g. `http://localhost:8888` |
|
|
28
|
+
| `checkpoints` | `true` | Save a checkpoint before the model changes files, so [`/undo`](usage.md#undo-and-checkpoints) can restore them |
|
|
28
29
|
|
|
29
30
|
Example file:
|
|
30
31
|
|
|
@@ -58,6 +59,7 @@ Environment variables override the file, which is useful for one-off runs and CI
|
|
|
58
59
|
| `~/.config/lcode/config.toml` | Settings |
|
|
59
60
|
| `~/.local/state/lcode/sessions/` | Saved conversations (for `lcode -c`) |
|
|
60
61
|
| `~/.local/state/lcode/history` | Prompt history (++up++ in the prompt) |
|
|
62
|
+
| `~/.local/state/lcode/checkpoints/` | Checkpoints for `/undo` (copies of your project's files; deleted after 14 days) |
|
|
61
63
|
| `~/.local/state/lcode/limits.json` | Context sizes that ran out of GPU memory on this machine (safe to delete) |
|
|
62
64
|
|
|
63
65
|
Sessions contain everything the model read, including file contents. Delete the folder to clear them.
|
|
@@ -75,6 +75,8 @@ the Ollama server you configure, it only contacts the web when the model searche
|
|
|
75
75
|
| `agent.py` | System prompt, streaming, tool loop, compaction, sessions |
|
|
76
76
|
| `tools.py` | Tool schemas and implementations |
|
|
77
77
|
| `permissions.py` | Approval prompts and the read-only allowlist |
|
|
78
|
+
| `bench.py` | `lcode bench`: the benchmark tasks, their checks and the reports |
|
|
79
|
+
| `checkpoints.py` | Snapshots before the model changes files, for `/undo` and `/rewind` |
|
|
78
80
|
| `catalog.py`, `models.toml` | Model catalog and memory estimates |
|
|
79
81
|
| `hardware.py` | NVIDIA / Apple Silicon detection |
|
|
80
82
|
| `ollama.py` | Ollama HTTP client |
|
|
@@ -67,7 +67,7 @@ until the task is done, asking before it changes anything.
|
|
|
67
67
|
---
|
|
68
68
|
|
|
69
69
|
Every edit is shown as a diff and every non-read-only command needs your approval, unless you
|
|
70
|
-
choose otherwise.
|
|
70
|
+
choose otherwise. `/undo` takes back a request's file changes.
|
|
71
71
|
|
|
72
72
|
- :material-swap-horizontal:{ .lg .middle } **Bring your own model**
|
|
73
73
|
|
|
@@ -17,8 +17,8 @@ size, and it works with any other Ollama model that supports tool calling.
|
|
|
17
17
|
| `qwen3.5-4b` | Qwen3.5 4B | dense | 3.4 GB | 256K | tested |
|
|
18
18
|
|
|
19
19
|
*Tested* means the model passed lcode's two acceptance tasks end to end (see
|
|
20
|
-
[test results](#test-results)). Results on other hardware are very welcome:
|
|
21
|
-
[report how a model does](https://github.com/nasser1941/lcode/issues/new?template=model_request.yml).
|
|
20
|
+
[test results](#test-results)). Results on other hardware are very welcome: run
|
|
21
|
+
[`lcode bench`](bench.md) and [report how a model does](https://github.com/nasser1941/lcode/issues/new?template=model_request.yml).
|
|
22
22
|
|
|
23
23
|
Run `lcode models` to see the same list with what fits on **your** machine:
|
|
24
24
|
|
|
@@ -8,6 +8,7 @@ lcode setup [model] # pick, download and configure a model
|
|
|
8
8
|
lcode models # the model catalog and what fits this machine
|
|
9
9
|
lcode doctor # check the installation (include this in bug reports)
|
|
10
10
|
lcode config [set|unset] # show or change settings
|
|
11
|
+
lcode bench [models] # score models on small coding tasks on this machine
|
|
11
12
|
```
|
|
12
13
|
|
|
13
14
|
| Option | Meaning |
|
|
@@ -49,6 +50,9 @@ reasoning is on.
|
|
|
49
50
|
| `/clear` | Start a new conversation (the current one stays saved) |
|
|
50
51
|
| `/rename NAME` | Name the current session so it's easy to find later |
|
|
51
52
|
| `/resume [SESSION]` | Resume a saved session: pick from a list, or give its number or name; `/resume all` lists every folder |
|
|
53
|
+
| `/undo` | Undo the file changes of the last request ([details](#undo-and-checkpoints)) |
|
|
54
|
+
| `/rewind [N]` | Go back to before request N: its files, and optionally the conversation |
|
|
55
|
+
| `/checkpoints` | List the requests that changed files, and which files |
|
|
52
56
|
| `/compact [focus]` | Summarize the conversation to free context |
|
|
53
57
|
| `/context [size]` | Show how full the context window is and change its size: pick from a list with memory estimates, or give a size like `/context 128k` (`/ctx` is a shortcut) |
|
|
54
58
|
| `/model [name]`, `/models` | Switch model, list models |
|
|
@@ -97,6 +101,38 @@ or ++n++. Text after ++n++ goes to the model as instructions: `n run the tests w
|
|
|
97
101
|
In `yolo` mode the model can run any command as your user. Use it in a container, a VM or a
|
|
98
102
|
throwaway clone, not on a machine with data you care about.
|
|
99
103
|
|
|
104
|
+
## Undo and checkpoints
|
|
105
|
+
|
|
106
|
+
Before the model first changes anything in a request (an edit, a new file, or a shell command that
|
|
107
|
+
isn't read-only), lcode saves a checkpoint of your project. If the result isn't what you wanted,
|
|
108
|
+
`/undo` puts the files back:
|
|
109
|
+
|
|
110
|
+
```text
|
|
111
|
+
❯ /undo
|
|
112
|
+
Undoing request 3: switch the parser to the new tokenizer
|
|
113
|
+
restore src/parser.py
|
|
114
|
+
restore tests/test_parser.py
|
|
115
|
+
remove src/tokenizer_v2.py
|
|
116
|
+
Done: restored src/parser.py, tests/test_parser.py; removed src/tokenizer_v2.py.
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
- `/undo` again goes one more request back. The model is told which files were restored, so it
|
|
120
|
+
reads them again instead of assuming its changes are still there.
|
|
121
|
+
- `/checkpoints` lists the requests that changed files. `/rewind 2` goes back to before request 2
|
|
122
|
+
in one step (undoing every request after it too) and offers to remove those requests from the
|
|
123
|
+
conversation as well.
|
|
124
|
+
- Changes made by shell commands are covered too (`sed -i`, generated files, `rm`), in any folder,
|
|
125
|
+
whether or not it's a git repository.
|
|
126
|
+
- If you edited one of those files yourself after the model did, lcode lists it and asks before
|
|
127
|
+
going back, since that would lose your edit. Files the model didn't touch are never changed.
|
|
128
|
+
|
|
129
|
+
Checkpoints are stored in a separate git repository under `~/.local/state/lcode/checkpoints/`. Your
|
|
130
|
+
own repository (its index, branches, stash and history) is never touched. Ignored files (per
|
|
131
|
+
`.gitignore`), `node_modules`, virtual environments and files over 10 MB aren't included. Folders
|
|
132
|
+
with more than 20,000 files, or your home folder itself, aren't checkpointed; lcode says so once and
|
|
133
|
+
carries on. Checkpoints are deleted after 14 days. To turn them off:
|
|
134
|
+
`lcode config set checkpoints false`.
|
|
135
|
+
|
|
100
136
|
## Web search
|
|
101
137
|
|
|
102
138
|
When the answer depends on information newer than the model's training data, such as the latest
|
|
@@ -19,6 +19,7 @@ from rich.panel import Panel
|
|
|
19
19
|
from rich.text import Text
|
|
20
20
|
|
|
21
21
|
from lcode import catalog, limits, sessions, web
|
|
22
|
+
from lcode.checkpoints import Checkpoints
|
|
22
23
|
from lcode.config import format_tokens
|
|
23
24
|
from lcode.ollama import Ollama, OllamaError
|
|
24
25
|
from lcode.permissions import Permissions
|
|
@@ -136,6 +137,7 @@ class Settings:
|
|
|
136
137
|
web: str = "on" # on | ask | off
|
|
137
138
|
search_backend: str = "auto"
|
|
138
139
|
searxng_url: str | None = None
|
|
140
|
+
checkpoints: bool = True # snapshot files before the model changes them, for /undo
|
|
139
141
|
|
|
140
142
|
|
|
141
143
|
class Agent:
|
|
@@ -147,10 +149,12 @@ class Agent:
|
|
|
147
149
|
self.perms = Permissions(self.console, settings.permission_mode)
|
|
148
150
|
self.tools = Toolbox(self)
|
|
149
151
|
self.session_id = self.new_session_id()
|
|
152
|
+
self.checkpoints = Checkpoints(self.console, settings.checkpoints)
|
|
150
153
|
self.session_name = ""
|
|
151
154
|
self.session_title = ""
|
|
152
155
|
self.ctx_used = 0
|
|
153
156
|
self.last_speed = 0.0
|
|
157
|
+
self.usage = {"requests": 0, "prompt_tokens": 0, "prompt_ns": 0, "output_tokens": 0, "output_ns": 0}
|
|
154
158
|
self.messages: list[dict] = []
|
|
155
159
|
self.reset()
|
|
156
160
|
|
|
@@ -220,6 +224,7 @@ class Agent:
|
|
|
220
224
|
"name": self.session_name,
|
|
221
225
|
"title": self.session_title,
|
|
222
226
|
"messages": self.messages,
|
|
227
|
+
"checkpoints": self.checkpoints.to_json(),
|
|
223
228
|
}
|
|
224
229
|
f.write_text(json.dumps(data))
|
|
225
230
|
|
|
@@ -228,6 +233,7 @@ class Agent:
|
|
|
228
233
|
self.session_id = self.new_session_id()
|
|
229
234
|
self.session_name = ""
|
|
230
235
|
self.session_title = ""
|
|
236
|
+
self.checkpoints.reset(self.session_id)
|
|
231
237
|
|
|
232
238
|
def rename(self, name: str) -> None:
|
|
233
239
|
self.session_name = " ".join(name.split())
|
|
@@ -249,6 +255,7 @@ class Agent:
|
|
|
249
255
|
note = f"The session's directory {saved_cwd} no longer exists; staying in {self.cwd}"
|
|
250
256
|
self.messages = [{"role": "system", "content": self.system_prompt()}, *data.get("messages", [])[1:]]
|
|
251
257
|
self.session_id = info.id
|
|
258
|
+
self.checkpoints.load(info.id, data.get("checkpoints") or [])
|
|
252
259
|
self.session_name = data.get("name", "")
|
|
253
260
|
self.session_title = data.get("title") or sessions.title_from(self.messages)
|
|
254
261
|
self.tools.read_mtimes.clear() # files may have changed since; the model must read them again
|
|
@@ -384,6 +391,11 @@ class Agent:
|
|
|
384
391
|
message["tool_calls"] = tool_calls
|
|
385
392
|
self.messages.append(message)
|
|
386
393
|
if final:
|
|
394
|
+
self.usage["requests"] += 1
|
|
395
|
+
self.usage["prompt_tokens"] += final.get("prompt_eval_count", 0)
|
|
396
|
+
self.usage["prompt_ns"] += final.get("prompt_eval_duration", 0)
|
|
397
|
+
self.usage["output_tokens"] += final.get("eval_count", 0)
|
|
398
|
+
self.usage["output_ns"] += final.get("eval_duration", 0)
|
|
387
399
|
self.ctx_used = final.get("prompt_eval_count", 0) + final.get("eval_count", 0)
|
|
388
400
|
seconds = final.get("eval_duration", 0) / 1e9
|
|
389
401
|
self.last_speed = final.get("eval_count", 0) / seconds if seconds else 0.0
|
|
@@ -406,6 +418,17 @@ class Agent:
|
|
|
406
418
|
return name
|
|
407
419
|
|
|
408
420
|
def run_turn(self, user_text: str) -> None:
|
|
421
|
+
self.checkpoints.begin_turn(self.session_id, user_text, len(self.messages))
|
|
422
|
+
try:
|
|
423
|
+
self._run_turn(user_text)
|
|
424
|
+
finally:
|
|
425
|
+
self.checkpoints.end_turn()
|
|
426
|
+
|
|
427
|
+
def checkpoint(self) -> None:
|
|
428
|
+
"""Called before the model changes files: snapshot them once per request, for /undo."""
|
|
429
|
+
self.checkpoints.before_change(self.cwd)
|
|
430
|
+
|
|
431
|
+
def _run_turn(self, user_text: str) -> None:
|
|
409
432
|
self.messages.append({"role": "user", "content": self.expand_mentions(user_text)})
|
|
410
433
|
malformed = 0
|
|
411
434
|
for _ in range(MAX_STEPS_PER_TURN):
|