errlore 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {errlore-0.1.3 → errlore-0.1.4}/CHANGELOG.md +22 -0
- {errlore-0.1.3 → errlore-0.1.4}/PKG-INFO +122 -20
- {errlore-0.1.3 → errlore-0.1.4}/README.md +121 -19
- {errlore-0.1.3 → errlore-0.1.4}/examples/anthropic_agent.py +4 -4
- errlore-0.1.4/examples/claude-code/README.md +37 -0
- {errlore-0.1.3 → errlore-0.1.4}/examples/langchain_agent.py +4 -3
- {errlore-0.1.3 → errlore-0.1.4}/examples/openai_agent.py +4 -3
- {errlore-0.1.3 → errlore-0.1.4}/pyproject.toml +4 -1
- {errlore-0.1.3 → errlore-0.1.4}/site/index.html +22 -19
- {errlore-0.1.3 → errlore-0.1.4}/site/llms.txt +9 -6
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/__init__.py +1 -1
- errlore-0.1.4/src/errlore/cli.py +200 -0
- errlore-0.1.4/src/errlore/integrations/__init__.py +5 -0
- errlore-0.1.4/src/errlore/integrations/claude_code.py +92 -0
- errlore-0.1.4/tests/test_cli.py +145 -0
- errlore-0.1.3/examples/claude-code/README.md +0 -23
- {errlore-0.1.3 → errlore-0.1.4}/.github/workflows/ci.yml +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/.gitignore +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/LICENSE +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/SECURITY.md +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/benchmarks/bench_error_reduction.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/benchmarks/bench_retrieval.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/benchmarks/results/error_reduction/report.md +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/examples/claude-code/errlore_posttooluse.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/examples/claude-code/errlore_sessionstart.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/examples/claude-code/settings.json.example +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/integrations/openwebui/README.md +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/integrations/openwebui/errlore_feedback_action.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/integrations/openwebui/errlore_memory_filter.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/site/demo.gif +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/site/demo_script.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/site/og.png +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/classifier.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/injector.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/patterns.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/tracker.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/facade.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/jsonl_index.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/jsonl_writer.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/repair.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/lessons/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/lessons/models.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/lessons/store.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/py.typed +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/retrieval/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/retrieval/backend.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/retrieval/index.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/sanitize.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/trust/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/src/errlore/trust/engine.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/conftest.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_errmem.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_facade.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_io.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_lessons.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_openwebui_integration.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_regressions.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_retrieval.py +0 -0
- {errlore-0.1.3 → errlore-0.1.4}/tests/test_trust.py +0 -0
|
@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.1.4] - 2026-07-09
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **CLI** (`errlore` console command): `errlore init claude-code` writes the
|
|
12
|
+
two Claude Code hook scripts and idempotently merges them into your
|
|
13
|
+
`settings.json` (global or `--project`), preserving existing hooks — a
|
|
14
|
+
one-command install instead of copy/edit/merge. Plus `errlore stats` and
|
|
15
|
+
`errlore lessons`.
|
|
16
|
+
- `errlore.integrations.claude_code` — the hook logic (`post_tool_use`,
|
|
17
|
+
`session_start`) now ships in the package and is tested, so the generated
|
|
18
|
+
hooks are 3-line shims.
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
- README: honest A/B framing (the knowledge-gap baseline fails by
|
|
22
|
+
construction; the result shows the capture-and-re-supply loop works, not
|
|
23
|
+
that memory teaches skills; single-run-at-temp-0 caveat). Coding-agent-first
|
|
24
|
+
hero. Security section reworded — the sanitizer is a noise filter on the
|
|
25
|
+
pattern, not an injection defense. Added a "Scale & limits" section
|
|
26
|
+
(unbounded injections journal, single-process trust/vector index).
|
|
27
|
+
- Examples no longer hard-code aging frontier model ids; they use
|
|
28
|
+
`os.getenv(...)` with a small default.
|
|
29
|
+
|
|
8
30
|
## [0.1.3] - 2026-07-06
|
|
9
31
|
|
|
10
32
|
### Fixed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: errlore
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server.
|
|
5
5
|
Project-URL: Homepage, https://errlore.com
|
|
6
6
|
Project-URL: Repository, https://github.com/Ma4etaSS/errlore
|
|
@@ -52,14 +52,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
|
|
|
52
52
|
into the prompt for similar future tasks.
|
|
53
53
|
- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
|
|
54
54
|
extraction tasks") injected as warnings.
|
|
55
|
-
- **Trust** -- Bayesian per-model, per-domain trust weights:
|
|
56
|
-
for which job, based on observed outcomes.
|
|
55
|
+
- **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
|
|
56
|
+
point for which model to pick per job, based on observed outcomes. Needs a spread of
|
|
57
|
+
real outcomes to separate models; shipped, but not yet proven on production traffic.
|
|
57
58
|
- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
|
|
58
59
|
reinforces or decays it automatically.
|
|
59
60
|
|
|
60
61
|
Embedded, file-based (JSONL), no server, no database, no API keys required.
|
|
61
62
|
Works fully offline. Your data never leaves your machine.
|
|
62
63
|
|
|
64
|
+
## Who it's for
|
|
65
|
+
|
|
66
|
+
errlore isn't memory for everything — it's memory for **failures**. It shines
|
|
67
|
+
wherever an agent repeats the *same class* of mistake:
|
|
68
|
+
|
|
69
|
+
- **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
|
|
70
|
+
the same bug or forgetting a project convention across sessions.
|
|
71
|
+
- **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
|
|
72
|
+
same date format, rounding rule, or schema field every week.
|
|
73
|
+
- **Any repeated-failure workflow** where a fix should stick the first time,
|
|
74
|
+
not be re-discovered on every run.
|
|
75
|
+
|
|
76
|
+
It fixes what the model doesn't *know* (a convention, a gotcha), not what it
|
|
77
|
+
*can't do* — see the benchmark below.
|
|
78
|
+
|
|
63
79
|
## Quickstart (< 5 minutes)
|
|
64
80
|
|
|
65
81
|
```bash
|
|
@@ -99,11 +115,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
|
|
|
99
115
|
|
|
100
116
|
## Does it actually reduce errors?
|
|
101
117
|
|
|
102
|
-
|
|
103
|
-
(`benchmarks/bench_error_reduction.py`): the same model
|
|
104
|
-
runs 96 tasks twice, with and without errlore injection.
|
|
105
|
-
validators, no LLM judges; raw outputs committed in
|
|
106
|
-
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/)
|
|
118
|
+
For the class of errors memory can fix — yes, and here's the honest version.
|
|
119
|
+
Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
|
|
120
|
+
(claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
|
|
121
|
+
Deterministic validators, no LLM judges; raw outputs committed in
|
|
122
|
+
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
|
|
123
|
+
you can recompute every number.
|
|
107
124
|
|
|
108
125
|
| arm | failures | fail rate |
|
|
109
126
|
|-----|----------|-----------|
|
|
@@ -114,14 +131,24 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
|
|
|
114
131
|
Split by error class:
|
|
115
132
|
|
|
116
133
|
- **Knowledge-gap errors** (workspace conventions: date formats, ID
|
|
117
|
-
normalization, rounding rules, CSV column order):
|
|
118
|
-
|
|
134
|
+
normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
|
|
135
|
+
can't know a convention it was never told, so arm A fails almost by
|
|
136
|
+
construction; the result shows errlore **captures the fix once and re-supplies
|
|
137
|
+
it** on the next similar task, end to end. That store-and-inject loop is the
|
|
138
|
+
claim — not that memory teaches skills.
|
|
119
139
|
- **Capability-gap errors** (letter counting, string reversal): 17/48 ->
|
|
120
140
|
20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
|
|
121
141
|
model doesn't know, not what it can't do.
|
|
122
142
|
|
|
123
|
-
|
|
124
|
-
|
|
143
|
+
**Caveats, up front:** this is a single run at temperature 0 (LLM output is
|
|
144
|
+
still slightly non-deterministic, so exact fine-grained counts vary between
|
|
145
|
+
runs — the large knowledge-gap effect is robust; the capability-gap delta is
|
|
146
|
+
within noise). The knowledge-gap task families use conventions the model
|
|
147
|
+
demonstrably can't guess, which is the point — but it means the headline is
|
|
148
|
+
"the loop works," not "90% fewer errors everywhere."
|
|
149
|
+
|
|
150
|
+
Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
|
|
151
|
+
(needs an Anthropic API key; task families and validators ship in the repo).
|
|
125
152
|
|
|
126
153
|
## How it works
|
|
127
154
|
|
|
@@ -146,13 +173,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
|
|
|
146
173
|
type before, `inject_for` adds a warning block to the prompt. Separate from
|
|
147
174
|
lessons: lessons are *solutions*, known issues are *warnings*.
|
|
148
175
|
|
|
149
|
-
### 3. Trust loop
|
|
176
|
+
### 3. Trust loop *(experimental)*
|
|
150
177
|
|
|
151
178
|
Bayesian per-model weights with adaptive learning rate, cold-start blending,
|
|
152
179
|
entropy enforcement, and temporal decay. After enough observations, call
|
|
153
180
|
`mem.best_model("code_generation")` to pick the model that historically
|
|
154
181
|
performs best on that domain.
|
|
155
182
|
|
|
183
|
+
> **Status: experimental.** The engine is tested and works, but discrimination
|
|
184
|
+
> between models only emerges from a *spread* of real outcomes over time — feed
|
|
185
|
+
> it a stream that is mostly successes and every model converges near the cap.
|
|
186
|
+
> Treat `best_model()` as a hint to validate, not a proven router yet. The
|
|
187
|
+
> lesson + known-issue loops above are the proven core (see the A/B benchmark).
|
|
188
|
+
|
|
156
189
|
## Semantic retrieval (optional)
|
|
157
190
|
|
|
158
191
|
By default, errlore finds relevant lessons via word overlap (zero
|
|
@@ -167,6 +200,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
|
|
|
167
200
|
mem = AgentMemory("./agent_memory", embeddings=True)
|
|
168
201
|
```
|
|
169
202
|
|
|
203
|
+
> The embedding model (~120 MB ONNX) is downloaded once on first use, then
|
|
204
|
+
> runs locally with no further network calls. The core (word-overlap) stays
|
|
205
|
+
> fully offline and dependency-free.
|
|
206
|
+
|
|
170
207
|
### Benchmark (adversarial paraphrasing)
|
|
171
208
|
|
|
172
209
|
Tested on 40 lessons with adversarially paraphrased queries
|
|
@@ -188,15 +225,37 @@ queries with shared vocabulary, word-overlap works fine.
|
|
|
188
225
|
errlore is framework-agnostic. It produces a text block; you put it in the
|
|
189
226
|
system prompt.
|
|
190
227
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
228
|
+
**Claude Code** — one command wires up failure-memory across sessions:
|
|
229
|
+
|
|
230
|
+
```bash
|
|
231
|
+
errlore init claude-code # or: --project for this repo only
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Failed Bash commands become lessons; every new session is briefed on past
|
|
235
|
+
pitfalls. See [examples/claude-code/](examples/claude-code/).
|
|
236
|
+
|
|
237
|
+
| Provider | Example |
|
|
238
|
+
|-------------|------------------------------------------------------|
|
|
239
|
+
| Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
|
|
240
|
+
| Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
|
|
241
|
+
| OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
|
|
242
|
+
| Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
|
|
243
|
+
| LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
|
|
196
244
|
|
|
197
|
-
|
|
245
|
+
The SDK examples run offline with `python examples/<name>.py` (mock responses,
|
|
198
246
|
no API keys). Set `use_api=True` to call real models.
|
|
199
247
|
|
|
248
|
+
### CLI
|
|
249
|
+
|
|
250
|
+
`pip install errlore` also installs an `errlore` command:
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
errlore init claude-code # install Claude Code hooks + settings
|
|
254
|
+
errlore stats # memory stats for a data dir (--data-dir)
|
|
255
|
+
errlore lessons # list stored lessons
|
|
256
|
+
errlore --version
|
|
257
|
+
```
|
|
258
|
+
|
|
200
259
|
## API overview
|
|
201
260
|
|
|
202
261
|
The main entry point is `AgentMemory`. All other classes are internal --
|
|
@@ -210,7 +269,7 @@ you only need them for advanced use.
|
|
|
210
269
|
| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
|
|
211
270
|
| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
|
|
212
271
|
| `lessons(limit)` | List all lessons (sorted by confidence). |
|
|
213
|
-
| `best_model(domain)` | Model with the highest trust weight
|
|
272
|
+
| `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
|
|
214
273
|
| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
|
|
215
274
|
| `pending_injections()` | Injections not yet reported. |
|
|
216
275
|
| `stats()` | Aggregate counts + trust weights. |
|
|
@@ -236,6 +295,49 @@ you only need them for advanced use.
|
|
|
236
295
|
(filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
|
|
237
296
|
(embedding metadata), `trust.json` (trust engine state).
|
|
238
297
|
|
|
298
|
+
## Security
|
|
299
|
+
|
|
300
|
+
A lesson is **trusted prompt content by design** — it is injected into your
|
|
301
|
+
prompts and reaches the model. So:
|
|
302
|
+
|
|
303
|
+
- **Do not ingest lessons from untrusted sources without review.** Treat lesson
|
|
304
|
+
capture like a code review, not like user input. A malicious lesson is a
|
|
305
|
+
prompt-injection vector — and this is the real control, not the sanitizer.
|
|
306
|
+
- **What the sanitizer does (and does not) do.** The lesson *pattern* passes
|
|
307
|
+
`sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
|
|
308
|
+
so log blobs don't pollute the prompt. It is a noise filter, **not** an
|
|
309
|
+
injection defense — it does not neutralize natural-language instructions, and
|
|
310
|
+
the *solution* text is stored as you author it (so it can hold real code).
|
|
311
|
+
Don't rely on it to make untrusted lessons safe.
|
|
312
|
+
- You control what becomes a lesson (`resolve(..., lesson=...)` /
|
|
313
|
+
`add_lesson(...)`); nothing is auto-promoted from raw model output.
|
|
314
|
+
|
|
315
|
+
Report security issues to the address in [SECURITY.md](SECURITY.md).
|
|
316
|
+
|
|
317
|
+
## Scale & limits (honest)
|
|
318
|
+
|
|
319
|
+
errlore is built for **one process, thousands of lessons** — a single agent or
|
|
320
|
+
a coding-agent session, not a high-throughput fleet. Know the edges:
|
|
321
|
+
|
|
322
|
+
- **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
|
|
323
|
+
ledger each call, so at very high injection volumes it slows down (roughly
|
|
324
|
+
linear in total injections). Fine for interactive/agent use; log compaction
|
|
325
|
+
is the next roadmap item. If you don't need the reinforcement loop, you can
|
|
326
|
+
ignore `report_outcome` and the file stays small.
|
|
327
|
+
- **Single-process by default.** The lesson/error stores use cross-process file
|
|
328
|
+
locks and are safe to share, but the **trust engine and the optional vector
|
|
329
|
+
index are not cross-process safe** — two processes writing `trust.json` /
|
|
330
|
+
`vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
|
|
331
|
+
writer, or give each process its own `data_dir`. Multi-agent shared memory is
|
|
332
|
+
on the roadmap.
|
|
333
|
+
- **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
|
|
334
|
+
over a large existing lesson store is slow the first time (then incremental).
|
|
335
|
+
- Concurrency is tested across threads; **multi-process** stress is not yet in
|
|
336
|
+
the suite.
|
|
337
|
+
|
|
338
|
+
None of these bite at the scale errlore targets today; they're stated so you
|
|
339
|
+
can decide, not discover.
|
|
340
|
+
|
|
239
341
|
## Roadmap
|
|
240
342
|
|
|
241
343
|
- [ ] Log compaction for injections journal
|
|
@@ -18,14 +18,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
|
|
|
18
18
|
into the prompt for similar future tasks.
|
|
19
19
|
- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
|
|
20
20
|
extraction tasks") injected as warnings.
|
|
21
|
-
- **Trust** -- Bayesian per-model, per-domain trust weights:
|
|
22
|
-
for which job, based on observed outcomes.
|
|
21
|
+
- **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
|
|
22
|
+
point for which model to pick per job, based on observed outcomes. Needs a spread of
|
|
23
|
+
real outcomes to separate models; shipped, but not yet proven on production traffic.
|
|
23
24
|
- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
|
|
24
25
|
reinforces or decays it automatically.
|
|
25
26
|
|
|
26
27
|
Embedded, file-based (JSONL), no server, no database, no API keys required.
|
|
27
28
|
Works fully offline. Your data never leaves your machine.
|
|
28
29
|
|
|
30
|
+
## Who it's for
|
|
31
|
+
|
|
32
|
+
errlore isn't memory for everything — it's memory for **failures**. It shines
|
|
33
|
+
wherever an agent repeats the *same class* of mistake:
|
|
34
|
+
|
|
35
|
+
- **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
|
|
36
|
+
the same bug or forgetting a project convention across sessions.
|
|
37
|
+
- **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
|
|
38
|
+
same date format, rounding rule, or schema field every week.
|
|
39
|
+
- **Any repeated-failure workflow** where a fix should stick the first time,
|
|
40
|
+
not be re-discovered on every run.
|
|
41
|
+
|
|
42
|
+
It fixes what the model doesn't *know* (a convention, a gotcha), not what it
|
|
43
|
+
*can't do* — see the benchmark below.
|
|
44
|
+
|
|
29
45
|
## Quickstart (< 5 minutes)
|
|
30
46
|
|
|
31
47
|
```bash
|
|
@@ -65,11 +81,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
|
|
|
65
81
|
|
|
66
82
|
## Does it actually reduce errors?
|
|
67
83
|
|
|
68
|
-
|
|
69
|
-
(`benchmarks/bench_error_reduction.py`): the same model
|
|
70
|
-
runs 96 tasks twice, with and without errlore injection.
|
|
71
|
-
validators, no LLM judges; raw outputs committed in
|
|
72
|
-
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/)
|
|
84
|
+
For the class of errors memory can fix — yes, and here's the honest version.
|
|
85
|
+
Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
|
|
86
|
+
(claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
|
|
87
|
+
Deterministic validators, no LLM judges; raw outputs committed in
|
|
88
|
+
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
|
|
89
|
+
you can recompute every number.
|
|
73
90
|
|
|
74
91
|
| arm | failures | fail rate |
|
|
75
92
|
|-----|----------|-----------|
|
|
@@ -80,14 +97,24 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
|
|
|
80
97
|
Split by error class:
|
|
81
98
|
|
|
82
99
|
- **Knowledge-gap errors** (workspace conventions: date formats, ID
|
|
83
|
-
normalization, rounding rules, CSV column order):
|
|
84
|
-
|
|
100
|
+
normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
|
|
101
|
+
can't know a convention it was never told, so arm A fails almost by
|
|
102
|
+
construction; the result shows errlore **captures the fix once and re-supplies
|
|
103
|
+
it** on the next similar task, end to end. That store-and-inject loop is the
|
|
104
|
+
claim — not that memory teaches skills.
|
|
85
105
|
- **Capability-gap errors** (letter counting, string reversal): 17/48 ->
|
|
86
106
|
20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
|
|
87
107
|
model doesn't know, not what it can't do.
|
|
88
108
|
|
|
89
|
-
|
|
90
|
-
|
|
109
|
+
**Caveats, up front:** this is a single run at temperature 0 (LLM output is
|
|
110
|
+
still slightly non-deterministic, so exact fine-grained counts vary between
|
|
111
|
+
runs — the large knowledge-gap effect is robust; the capability-gap delta is
|
|
112
|
+
within noise). The knowledge-gap task families use conventions the model
|
|
113
|
+
demonstrably can't guess, which is the point — but it means the headline is
|
|
114
|
+
"the loop works," not "90% fewer errors everywhere."
|
|
115
|
+
|
|
116
|
+
Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
|
|
117
|
+
(needs an Anthropic API key; task families and validators ship in the repo).
|
|
91
118
|
|
|
92
119
|
## How it works
|
|
93
120
|
|
|
@@ -112,13 +139,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
|
|
|
112
139
|
type before, `inject_for` adds a warning block to the prompt. Separate from
|
|
113
140
|
lessons: lessons are *solutions*, known issues are *warnings*.
|
|
114
141
|
|
|
115
|
-
### 3. Trust loop
|
|
142
|
+
### 3. Trust loop *(experimental)*
|
|
116
143
|
|
|
117
144
|
Bayesian per-model weights with adaptive learning rate, cold-start blending,
|
|
118
145
|
entropy enforcement, and temporal decay. After enough observations, call
|
|
119
146
|
`mem.best_model("code_generation")` to pick the model that historically
|
|
120
147
|
performs best on that domain.
|
|
121
148
|
|
|
149
|
+
> **Status: experimental.** The engine is tested and works, but discrimination
|
|
150
|
+
> between models only emerges from a *spread* of real outcomes over time — feed
|
|
151
|
+
> it a stream that is mostly successes and every model converges near the cap.
|
|
152
|
+
> Treat `best_model()` as a hint to validate, not a proven router yet. The
|
|
153
|
+
> lesson + known-issue loops above are the proven core (see the A/B benchmark).
|
|
154
|
+
|
|
122
155
|
## Semantic retrieval (optional)
|
|
123
156
|
|
|
124
157
|
By default, errlore finds relevant lessons via word overlap (zero
|
|
@@ -133,6 +166,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
|
|
|
133
166
|
mem = AgentMemory("./agent_memory", embeddings=True)
|
|
134
167
|
```
|
|
135
168
|
|
|
169
|
+
> The embedding model (~120 MB ONNX) is downloaded once on first use, then
|
|
170
|
+
> runs locally with no further network calls. The core (word-overlap) stays
|
|
171
|
+
> fully offline and dependency-free.
|
|
172
|
+
|
|
136
173
|
### Benchmark (adversarial paraphrasing)
|
|
137
174
|
|
|
138
175
|
Tested on 40 lessons with adversarially paraphrased queries
|
|
@@ -154,15 +191,37 @@ queries with shared vocabulary, word-overlap works fine.
|
|
|
154
191
|
errlore is framework-agnostic. It produces a text block; you put it in the
|
|
155
192
|
system prompt.
|
|
156
193
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
194
|
+
**Claude Code** — one command wires up failure-memory across sessions:
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
errlore init claude-code # or: --project for this repo only
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Failed Bash commands become lessons; every new session is briefed on past
|
|
201
|
+
pitfalls. See [examples/claude-code/](examples/claude-code/).
|
|
202
|
+
|
|
203
|
+
| Provider | Example |
|
|
204
|
+
|-------------|------------------------------------------------------|
|
|
205
|
+
| Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
|
|
206
|
+
| Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
|
|
207
|
+
| OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
|
|
208
|
+
| Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
|
|
209
|
+
| LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
|
|
162
210
|
|
|
163
|
-
|
|
211
|
+
The SDK examples run offline with `python examples/<name>.py` (mock responses,
|
|
164
212
|
no API keys). Set `use_api=True` to call real models.
|
|
165
213
|
|
|
214
|
+
### CLI
|
|
215
|
+
|
|
216
|
+
`pip install errlore` also installs an `errlore` command:
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
errlore init claude-code # install Claude Code hooks + settings
|
|
220
|
+
errlore stats # memory stats for a data dir (--data-dir)
|
|
221
|
+
errlore lessons # list stored lessons
|
|
222
|
+
errlore --version
|
|
223
|
+
```
|
|
224
|
+
|
|
166
225
|
## API overview
|
|
167
226
|
|
|
168
227
|
The main entry point is `AgentMemory`. All other classes are internal --
|
|
@@ -176,7 +235,7 @@ you only need them for advanced use.
|
|
|
176
235
|
| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
|
|
177
236
|
| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
|
|
178
237
|
| `lessons(limit)` | List all lessons (sorted by confidence). |
|
|
179
|
-
| `best_model(domain)` | Model with the highest trust weight
|
|
238
|
+
| `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
|
|
180
239
|
| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
|
|
181
240
|
| `pending_injections()` | Injections not yet reported. |
|
|
182
241
|
| `stats()` | Aggregate counts + trust weights. |
|
|
@@ -202,6 +261,49 @@ you only need them for advanced use.
|
|
|
202
261
|
(filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
|
|
203
262
|
(embedding metadata), `trust.json` (trust engine state).
|
|
204
263
|
|
|
264
|
+
## Security
|
|
265
|
+
|
|
266
|
+
A lesson is **trusted prompt content by design** — it is injected into your
|
|
267
|
+
prompts and reaches the model. So:
|
|
268
|
+
|
|
269
|
+
- **Do not ingest lessons from untrusted sources without review.** Treat lesson
|
|
270
|
+
capture like a code review, not like user input. A malicious lesson is a
|
|
271
|
+
prompt-injection vector — and this is the real control, not the sanitizer.
|
|
272
|
+
- **What the sanitizer does (and does not) do.** The lesson *pattern* passes
|
|
273
|
+
`sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
|
|
274
|
+
so log blobs don't pollute the prompt. It is a noise filter, **not** an
|
|
275
|
+
injection defense — it does not neutralize natural-language instructions, and
|
|
276
|
+
the *solution* text is stored as you author it (so it can hold real code).
|
|
277
|
+
Don't rely on it to make untrusted lessons safe.
|
|
278
|
+
- You control what becomes a lesson (`resolve(..., lesson=...)` /
|
|
279
|
+
`add_lesson(...)`); nothing is auto-promoted from raw model output.
|
|
280
|
+
|
|
281
|
+
Report security issues to the address in [SECURITY.md](SECURITY.md).
|
|
282
|
+
|
|
283
|
+
## Scale & limits (honest)
|
|
284
|
+
|
|
285
|
+
errlore is built for **one process, thousands of lessons** — a single agent or
|
|
286
|
+
a coding-agent session, not a high-throughput fleet. Know the edges:
|
|
287
|
+
|
|
288
|
+
- **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
|
|
289
|
+
ledger each call, so at very high injection volumes it slows down (roughly
|
|
290
|
+
linear in total injections). Fine for interactive/agent use; log compaction
|
|
291
|
+
is the next roadmap item. If you don't need the reinforcement loop, you can
|
|
292
|
+
ignore `report_outcome` and the file stays small.
|
|
293
|
+
- **Single-process by default.** The lesson/error stores use cross-process file
|
|
294
|
+
locks and are safe to share, but the **trust engine and the optional vector
|
|
295
|
+
index are not cross-process safe** — two processes writing `trust.json` /
|
|
296
|
+
`vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
|
|
297
|
+
writer, or give each process its own `data_dir`. Multi-agent shared memory is
|
|
298
|
+
on the roadmap.
|
|
299
|
+
- **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
|
|
300
|
+
over a large existing lesson store is slow the first time (then incremental).
|
|
301
|
+
- Concurrency is tested across threads; **multi-process** stress is not yet in
|
|
302
|
+
the suite.
|
|
303
|
+
|
|
304
|
+
None of these bite at the scale errlore targets today; they're stated so you
|
|
305
|
+
can decide, not discover.
|
|
306
|
+
|
|
205
307
|
## Roadmap
|
|
206
308
|
|
|
207
309
|
- [ ] Log compaction for injections journal
|
|
@@ -16,15 +16,15 @@ runs end-to-end without network access.
|
|
|
16
16
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
+
import os
|
|
19
20
|
import tempfile
|
|
20
21
|
from pathlib import Path
|
|
21
22
|
|
|
22
23
|
from errlore import AgentMemory
|
|
23
24
|
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
|
|
27
|
-
MODEL = "claude-opus-4-8"
|
|
25
|
+
# Model is just a label to errlore (it never calls the API itself). Override
|
|
26
|
+
# with your own Claude model id; the default is a small, fast, cheap option.
|
|
27
|
+
MODEL = os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-latest")
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
# -- Agent wrapper -----------------------------------------------------------
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# errlore + Claude Code
|
|
2
|
+
|
|
3
|
+
Give your coding agent a memory of its own failures across sessions:
|
|
4
|
+
|
|
5
|
+
- **PostToolUse hook** — every failed Bash command is logged into errlore.
|
|
6
|
+
Resolve the ones you fixed (`mem.resolve(err_id, ..., lesson=...)`) or use
|
|
7
|
+
`mem.add_lesson()` to capture takeaways directly.
|
|
8
|
+
- **SessionStart hook** — each new session begins with a briefing block of
|
|
9
|
+
relevant lessons and per-tool KNOWN ISSUES, printed into the context.
|
|
10
|
+
|
|
11
|
+
## Setup (one command)
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install errlore
|
|
15
|
+
errlore init claude-code # global (~/.claude/settings.json)
|
|
16
|
+
errlore init claude-code --project # or this repo only (./.claude/settings.json)
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
That writes the two hook scripts (to `~/.errlore/hooks/`) and merges them into
|
|
20
|
+
your `settings.json` — idempotently, preserving any hooks you already have.
|
|
21
|
+
Restart Claude Code (or open a new session) to pick them up. Options:
|
|
22
|
+
`--data-dir` (where the memory lives, default `~/.errlore/claude-code`) and
|
|
23
|
+
`--hooks-dir`.
|
|
24
|
+
|
|
25
|
+
Handy afterwards: `errlore stats` and `errlore lessons` to see what it learned.
|
|
26
|
+
|
|
27
|
+
### Manual setup (if you'd rather wire it yourself)
|
|
28
|
+
|
|
29
|
+
The scripts in this folder (`errlore_posttooluse.py`, `errlore_sessionstart.py`)
|
|
30
|
+
plus `settings.json.example` show the shape: copy them somewhere stable, fix the
|
|
31
|
+
paths, and merge into `.claude/settings.json`. `export ERRLORE_DATA=...` picks
|
|
32
|
+
the memory dir.
|
|
33
|
+
|
|
34
|
+
Notes: hook event field names can differ between Claude Code versions —
|
|
35
|
+
the PostToolUse script reads them defensively and never breaks the agent
|
|
36
|
+
loop (exit 0 on anything unexpected). Check `claude --help` / the hooks
|
|
37
|
+
docs for your version if events don't arrive.
|
|
@@ -19,14 +19,15 @@ The ``if __name__`` block uses a mock instead of a real LLM call.
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
+
import os
|
|
22
23
|
import tempfile
|
|
23
24
|
from pathlib import Path
|
|
24
25
|
|
|
25
26
|
from errlore import AgentMemory, Injection
|
|
26
27
|
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
MODEL = "gpt-
|
|
28
|
+
# Model is just a label to errlore (it never calls the API itself). Override
|
|
29
|
+
# with your own; the default is a small, cheap, widely-available option.
|
|
30
|
+
MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
# -- errlore + LangChain glue ------------------------------------------------
|
|
@@ -16,14 +16,15 @@ runs end-to-end without network access.
|
|
|
16
16
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
+
import os
|
|
19
20
|
import tempfile
|
|
20
21
|
from pathlib import Path
|
|
21
22
|
|
|
22
23
|
from errlore import AgentMemory
|
|
23
24
|
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
MODEL = "gpt-
|
|
25
|
+
# Model is just a label to errlore (it never calls the API itself). Override
|
|
26
|
+
# with your own; the default is a small, cheap, widely-available option.
|
|
27
|
+
MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
# -- Agent wrapper -----------------------------------------------------------
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "errlore"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.4"
|
|
8
8
|
description = "Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -24,6 +24,9 @@ dependencies = [
|
|
|
24
24
|
"filelock>=3.13",
|
|
25
25
|
]
|
|
26
26
|
|
|
27
|
+
[project.scripts]
|
|
28
|
+
errlore = "errlore.cli:main"
|
|
29
|
+
|
|
27
30
|
[project.optional-dependencies]
|
|
28
31
|
embeddings = [
|
|
29
32
|
"fastembed>=0.6,<0.9",
|