errlore 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {errlore-0.1.2 → errlore-0.1.4}/CHANGELOG.md +39 -0
- {errlore-0.1.2 → errlore-0.1.4}/PKG-INFO +140 -12
- errlore-0.1.4/README.md +318 -0
- {errlore-0.1.2 → errlore-0.1.4}/examples/anthropic_agent.py +4 -4
- errlore-0.1.4/examples/claude-code/README.md +37 -0
- {errlore-0.1.2 → errlore-0.1.4}/examples/langchain_agent.py +4 -3
- {errlore-0.1.2 → errlore-0.1.4}/examples/openai_agent.py +4 -3
- {errlore-0.1.2 → errlore-0.1.4}/pyproject.toml +4 -1
- {errlore-0.1.2 → errlore-0.1.4}/site/index.html +41 -18
- {errlore-0.1.2 → errlore-0.1.4}/site/llms.txt +10 -6
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/__init__.py +1 -1
- errlore-0.1.4/src/errlore/cli.py +200 -0
- errlore-0.1.4/src/errlore/integrations/__init__.py +5 -0
- errlore-0.1.4/src/errlore/integrations/claude_code.py +92 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/jsonl_writer.py +7 -8
- errlore-0.1.4/tests/test_cli.py +145 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_regressions.py +56 -0
- errlore-0.1.2/README.md +0 -190
- errlore-0.1.2/examples/claude-code/README.md +0 -23
- {errlore-0.1.2 → errlore-0.1.4}/.github/workflows/ci.yml +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/.gitignore +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/LICENSE +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/SECURITY.md +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/benchmarks/bench_error_reduction.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/benchmarks/bench_retrieval.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/benchmarks/results/error_reduction/report.md +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/examples/claude-code/errlore_posttooluse.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/examples/claude-code/errlore_sessionstart.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/examples/claude-code/settings.json.example +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/integrations/openwebui/README.md +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/integrations/openwebui/errlore_feedback_action.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/integrations/openwebui/errlore_memory_filter.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/site/demo.gif +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/site/demo_script.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/site/og.png +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/__init__.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/classifier.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/injector.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/patterns.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/tracker.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/facade.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/__init__.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/jsonl_index.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/repair.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/lessons/__init__.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/lessons/models.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/lessons/store.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/py.typed +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/retrieval/__init__.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/retrieval/backend.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/retrieval/index.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/sanitize.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/trust/__init__.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/src/errlore/trust/engine.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/conftest.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_errmem.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_facade.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_io.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_lessons.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_openwebui_integration.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_retrieval.py +0 -0
- {errlore-0.1.2 → errlore-0.1.4}/tests/test_trust.py +0 -0
|
@@ -5,6 +5,45 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.1.4] - 2026-07-09
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **CLI** (`errlore` console command): `errlore init claude-code` writes the
|
|
12
|
+
two Claude Code hook scripts and idempotently merges them into your
|
|
13
|
+
`settings.json` (global or `--project`), preserving existing hooks — a
|
|
14
|
+
one-command install instead of copy/edit/merge. Plus `errlore stats` and
|
|
15
|
+
`errlore lessons`.
|
|
16
|
+
- `errlore.integrations.claude_code` — the hook logic (`post_tool_use`,
|
|
17
|
+
`session_start`) now ships in the package and is tested, so the generated
|
|
18
|
+
hooks are 3-line shims.
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
- README: honest A/B framing (the knowledge-gap baseline fails by
|
|
22
|
+
construction; the result shows the capture-and-re-supply loop works, not
|
|
23
|
+
that memory teaches skills; single-run-at-temp-0 caveat). Coding-agent-first
|
|
24
|
+
hero. Security section reworded — the sanitizer is a noise filter on the
|
|
25
|
+
pattern, not an injection defense. Added a "Scale & limits" section
|
|
26
|
+
(unbounded injections journal, single-process trust/vector index).
|
|
27
|
+
- Examples no longer hard-code aging frontier model ids; they use
|
|
28
|
+
`os.getenv(...)` with a small default.
|
|
29
|
+
|
|
30
|
+
## [0.1.3] - 2026-07-06
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
- Read-cache poisoning race in `JSONLWriter.read_all`: an append landing
|
|
34
|
+
mid-parse cached pre-append records under the post-append mtime/size, so
|
|
35
|
+
later reads returned stale data and `atomic_update` could silently drop
|
|
36
|
+
the concurrent record. Now caches only when the file is unchanged across
|
|
37
|
+
the parse (pre/post stat match). Found by pre-launch adversarial audit;
|
|
38
|
+
regression test added (Bug 3).
|
|
39
|
+
- Removed dead `JSONLWriter._invalidate_cache`.
|
|
40
|
+
|
|
41
|
+
### Changed
|
|
42
|
+
- README: added the error-reduction A/B benchmark section (96 paired tasks,
|
|
43
|
+
63 -> 20 failures, McNemar p=1.8e-09; knowledge-gap 46/48 -> 0/48,
|
|
44
|
+
capability-gap honestly worse at 17/48 -> 20/48), so the PyPI page shows
|
|
45
|
+
the headline evidence, not just the retrieval table.
|
|
46
|
+
|
|
8
47
|
## [0.1.2] - 2026-07-06
|
|
9
48
|
|
|
10
49
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: errlore
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server.
|
|
5
5
|
Project-URL: Homepage, https://errlore.com
|
|
6
6
|
Project-URL: Repository, https://github.com/Ma4etaSS/errlore
|
|
@@ -52,14 +52,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
|
|
|
52
52
|
into the prompt for similar future tasks.
|
|
53
53
|
- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
|
|
54
54
|
extraction tasks") injected as warnings.
|
|
55
|
-
- **Trust** -- Bayesian per-model, per-domain trust weights:
|
|
56
|
-
for which job, based on observed outcomes.
|
|
55
|
+
- **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
|
|
56
|
+
point for which model to pick per job, based on observed outcomes. Needs a spread of
|
|
57
|
+
real outcomes to separate models; shipped, but not yet proven on production traffic.
|
|
57
58
|
- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
|
|
58
59
|
reinforces or decays it automatically.
|
|
59
60
|
|
|
60
61
|
Embedded, file-based (JSONL), no server, no database, no API keys required.
|
|
61
62
|
Works fully offline. Your data never leaves your machine.
|
|
62
63
|
|
|
64
|
+
## Who it's for
|
|
65
|
+
|
|
66
|
+
errlore isn't memory for everything — it's memory for **failures**. It shines
|
|
67
|
+
wherever an agent repeats the *same class* of mistake:
|
|
68
|
+
|
|
69
|
+
- **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
|
|
70
|
+
the same bug or forgetting a project convention across sessions.
|
|
71
|
+
- **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
|
|
72
|
+
same date format, rounding rule, or schema field every week.
|
|
73
|
+
- **Any repeated-failure workflow** where a fix should stick the first time,
|
|
74
|
+
not be re-discovered on every run.
|
|
75
|
+
|
|
76
|
+
It fixes what the model doesn't *know* (a convention, a gotcha), not what it
|
|
77
|
+
*can't do* — see the benchmark below.
|
|
78
|
+
|
|
63
79
|
## Quickstart (< 5 minutes)
|
|
64
80
|
|
|
65
81
|
```bash
|
|
@@ -91,12 +107,49 @@ mem.report_outcome(inj, success=True)
|
|
|
91
107
|
print(mem.stats())
|
|
92
108
|
# {'errors_total': 1, 'errors_resolved': 1, 'errors_unresolved': 0,
|
|
93
109
|
# 'lessons_total': 1, 'lessons_applied': 1, 'pending_injections': 0,
|
|
94
|
-
# 'trust': {'gpt-5.5': 0.
|
|
110
|
+
# 'trust': {'gpt-5.5': 0.5522...}}
|
|
95
111
|
```
|
|
96
112
|
|
|
97
113
|
No API keys needed. errlore itself never calls any LLM -- it manages local
|
|
98
114
|
JSONL files and does text matching. LLM calls are yours to make (or not).
|
|
99
115
|
|
|
116
|
+
## Does it actually reduce errors?
|
|
117
|
+
|
|
118
|
+
For the class of errors memory can fix — yes, and here's the honest version.
|
|
119
|
+
Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
|
|
120
|
+
(claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
|
|
121
|
+
Deterministic validators, no LLM judges; raw outputs committed in
|
|
122
|
+
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
|
|
123
|
+
you can recompute every number.
|
|
124
|
+
|
|
125
|
+
| arm | failures | fail rate |
|
|
126
|
+
|-----|----------|-----------|
|
|
127
|
+
| A: plain | 63/96 | 65.6% |
|
|
128
|
+
| B: with errlore | 20/96 | 20.8% |
|
|
129
|
+
|
|
130
|
+
Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
|
|
131
|
+
Split by error class:
|
|
132
|
+
|
|
133
|
+
- **Knowledge-gap errors** (workspace conventions: date formats, ID
|
|
134
|
+
normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
|
|
135
|
+
can't know a convention it was never told, so arm A fails almost by
|
|
136
|
+
construction; the result shows errlore **captures the fix once and re-supplies
|
|
137
|
+
it** on the next similar task, end to end. That store-and-inject loop is the
|
|
138
|
+
claim — not that memory teaches skills.
|
|
139
|
+
- **Capability-gap errors** (letter counting, string reversal): 17/48 ->
|
|
140
|
+
20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
|
|
141
|
+
model doesn't know, not what it can't do.
|
|
142
|
+
|
|
143
|
+
**Caveats, up front:** this is a single run at temperature 0 (LLM output is
|
|
144
|
+
still slightly non-deterministic, so exact fine-grained counts vary between
|
|
145
|
+
runs — the large knowledge-gap effect is robust; the capability-gap delta is
|
|
146
|
+
within noise). The knowledge-gap task families use conventions the model
|
|
147
|
+
demonstrably can't guess, which is the point — but it means the headline is
|
|
148
|
+
"the loop works," not "90% fewer errors everywhere."
|
|
149
|
+
|
|
150
|
+
Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
|
|
151
|
+
(needs an Anthropic API key; task families and validators ship in the repo).
|
|
152
|
+
|
|
100
153
|
## How it works
|
|
101
154
|
|
|
102
155
|
errlore runs three reinforcement loops around your agent:
|
|
@@ -120,13 +173,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
|
|
|
120
173
|
type before, `inject_for` adds a warning block to the prompt. Separate from
|
|
121
174
|
lessons: lessons are *solutions*, known issues are *warnings*.
|
|
122
175
|
|
|
123
|
-
### 3. Trust loop
|
|
176
|
+
### 3. Trust loop *(experimental)*
|
|
124
177
|
|
|
125
178
|
Bayesian per-model weights with adaptive learning rate, cold-start blending,
|
|
126
179
|
entropy enforcement, and temporal decay. After enough observations, call
|
|
127
180
|
`mem.best_model("code_generation")` to pick the model that historically
|
|
128
181
|
performs best on that domain.
|
|
129
182
|
|
|
183
|
+
> **Status: experimental.** The engine is tested and works, but discrimination
|
|
184
|
+
> between models only emerges from a *spread* of real outcomes over time — feed
|
|
185
|
+
> it a stream that is mostly successes and every model converges near the cap.
|
|
186
|
+
> Treat `best_model()` as a hint to validate, not a proven router yet. The
|
|
187
|
+
> lesson + known-issue loops above are the proven core (see the A/B benchmark).
|
|
188
|
+
|
|
130
189
|
## Semantic retrieval (optional)
|
|
131
190
|
|
|
132
191
|
By default, errlore finds relevant lessons via word overlap (zero
|
|
@@ -141,6 +200,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
|
|
|
141
200
|
mem = AgentMemory("./agent_memory", embeddings=True)
|
|
142
201
|
```
|
|
143
202
|
|
|
203
|
+
> The embedding model (~120 MB ONNX) is downloaded once on first use, then
|
|
204
|
+
> runs locally with no further network calls. The core (word-overlap) stays
|
|
205
|
+
> fully offline and dependency-free.
|
|
206
|
+
|
|
144
207
|
### Benchmark (adversarial paraphrasing)
|
|
145
208
|
|
|
146
209
|
Tested on 40 lessons with adversarially paraphrased queries
|
|
@@ -162,15 +225,37 @@ queries with shared vocabulary, word-overlap works fine.
|
|
|
162
225
|
errlore is framework-agnostic. It produces a text block; you put it in the
|
|
163
226
|
system prompt.
|
|
164
227
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
228
|
+
**Claude Code** — one command wires up failure-memory across sessions:
|
|
229
|
+
|
|
230
|
+
```bash
|
|
231
|
+
errlore init claude-code # or: --project for this repo only
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Failed Bash commands become lessons; every new session is briefed on past
|
|
235
|
+
pitfalls. See [examples/claude-code/](examples/claude-code/).
|
|
236
|
+
|
|
237
|
+
| Provider | Example |
|
|
238
|
+
|-------------|------------------------------------------------------|
|
|
239
|
+
| Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
|
|
240
|
+
| Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
|
|
241
|
+
| OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
|
|
242
|
+
| Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
|
|
243
|
+
| LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
|
|
170
244
|
|
|
171
|
-
|
|
245
|
+
The SDK examples run offline with `python examples/<name>.py` (mock responses,
|
|
172
246
|
no API keys). Set `use_api=True` to call real models.
|
|
173
247
|
|
|
248
|
+
### CLI
|
|
249
|
+
|
|
250
|
+
`pip install errlore` also installs an `errlore` command:
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
errlore init claude-code # install Claude Code hooks + settings
|
|
254
|
+
errlore stats # memory stats for a data dir (--data-dir)
|
|
255
|
+
errlore lessons # list stored lessons
|
|
256
|
+
errlore --version
|
|
257
|
+
```
|
|
258
|
+
|
|
174
259
|
## API overview
|
|
175
260
|
|
|
176
261
|
The main entry point is `AgentMemory`. All other classes are internal --
|
|
@@ -184,7 +269,7 @@ you only need them for advanced use.
|
|
|
184
269
|
| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
|
|
185
270
|
| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
|
|
186
271
|
| `lessons(limit)` | List all lessons (sorted by confidence). |
|
|
187
|
-
| `best_model(domain)` | Model with the highest trust weight
|
|
272
|
+
| `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
|
|
188
273
|
| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
|
|
189
274
|
| `pending_injections()` | Injections not yet reported. |
|
|
190
275
|
| `stats()` | Aggregate counts + trust weights. |
|
|
@@ -210,6 +295,49 @@ you only need them for advanced use.
|
|
|
210
295
|
(filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
|
|
211
296
|
(embedding metadata), `trust.json` (trust engine state).
|
|
212
297
|
|
|
298
|
+
## Security
|
|
299
|
+
|
|
300
|
+
A lesson is **trusted prompt content by design** — it is injected into your
|
|
301
|
+
prompts and reaches the model. So:
|
|
302
|
+
|
|
303
|
+
- **Do not ingest lessons from untrusted sources without review.** Treat lesson
|
|
304
|
+
capture like a code review, not like user input. A malicious lesson is a
|
|
305
|
+
prompt-injection vector — and this is the real control, not the sanitizer.
|
|
306
|
+
- **What the sanitizer does (and does not) do.** The lesson *pattern* passes
|
|
307
|
+
`sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
|
|
308
|
+
so log blobs don't pollute the prompt. It is a noise filter, **not** an
|
|
309
|
+
injection defense — it does not neutralize natural-language instructions, and
|
|
310
|
+
the *solution* text is stored as you author it (so it can hold real code).
|
|
311
|
+
Don't rely on it to make untrusted lessons safe.
|
|
312
|
+
- You control what becomes a lesson (`resolve(..., lesson=...)` /
|
|
313
|
+
`add_lesson(...)`); nothing is auto-promoted from raw model output.
|
|
314
|
+
|
|
315
|
+
Report security issues to the address in [SECURITY.md](SECURITY.md).
|
|
316
|
+
|
|
317
|
+
## Scale & limits (honest)
|
|
318
|
+
|
|
319
|
+
errlore is built for **one process, thousands of lessons** — a single agent or
|
|
320
|
+
a coding-agent session, not a high-throughput fleet. Know the edges:
|
|
321
|
+
|
|
322
|
+
- **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
|
|
323
|
+
ledger each call, so at very high injection volumes it slows down (roughly
|
|
324
|
+
linear in total injections). Fine for interactive/agent use; log compaction
|
|
325
|
+
is the next roadmap item. If you don't need the reinforcement loop, you can
|
|
326
|
+
ignore `report_outcome` and the file stays small.
|
|
327
|
+
- **Single-process by default.** The lesson/error stores use cross-process file
|
|
328
|
+
locks and are safe to share, but the **trust engine and the optional vector
|
|
329
|
+
index are not cross-process safe** — two processes writing `trust.json` /
|
|
330
|
+
`vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
|
|
331
|
+
writer, or give each process its own `data_dir`. Multi-agent shared memory is
|
|
332
|
+
on the roadmap.
|
|
333
|
+
- **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
|
|
334
|
+
over a large existing lesson store is slow the first time (then incremental).
|
|
335
|
+
- Concurrency is tested across threads; **multi-process** stress is not yet in
|
|
336
|
+
the suite.
|
|
337
|
+
|
|
338
|
+
None of these bite at the scale errlore targets today; they're stated so you
|
|
339
|
+
can decide, not discover.
|
|
340
|
+
|
|
213
341
|
## Roadmap
|
|
214
342
|
|
|
215
343
|
- [ ] Log compaction for injections journal
|
errlore-0.1.4/README.md
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
# errlore
|
|
2
|
+
|
|
3
|
+
**Memory for AI agents that learns from failures.**
|
|
4
|
+
|
|
5
|
+
*Stop the second mistake, not just the first.*
|
|
6
|
+
|
|
7
|
+

|
|
8
|
+
|
|
9
|
+
[](https://github.com/Ma4etaSS/errlore/actions/workflows/ci.yml)
|
|
10
|
+
[](https://python.org)
|
|
11
|
+
[](LICENSE)
|
|
12
|
+
|
|
13
|
+
Extracted from a 324K LOC production multi-LLM orchestration system, keeping the one part that demonstrably worked: the error-memory loop that made agents stop repeating mistakes.
|
|
14
|
+
|
|
15
|
+
Your agent keeps making the same mistakes. errlore fixes that:
|
|
16
|
+
|
|
17
|
+
- **Lessons** -- every resolved failure becomes a lesson; relevant lessons are injected
|
|
18
|
+
into the prompt for similar future tasks.
|
|
19
|
+
- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
|
|
20
|
+
extraction tasks") injected as warnings.
|
|
21
|
+
- **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
|
|
22
|
+
point for which model to pick per job, based on observed outcomes. Needs a spread of
|
|
23
|
+
real outcomes to separate models; shipped, but not yet proven on production traffic.
|
|
24
|
+
- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
|
|
25
|
+
reinforces or decays it automatically.
|
|
26
|
+
|
|
27
|
+
Embedded, file-based (JSONL), no server, no database, no API keys required.
|
|
28
|
+
Works fully offline. Your data never leaves your machine.
|
|
29
|
+
|
|
30
|
+
## Who it's for
|
|
31
|
+
|
|
32
|
+
errlore isn't memory for everything — it's memory for **failures**. It shines
|
|
33
|
+
wherever an agent repeats the *same class* of mistake:
|
|
34
|
+
|
|
35
|
+
- **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
|
|
36
|
+
the same bug or forgetting a project convention across sessions.
|
|
37
|
+
- **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
|
|
38
|
+
same date format, rounding rule, or schema field every week.
|
|
39
|
+
- **Any repeated-failure workflow** where a fix should stick the first time,
|
|
40
|
+
not be re-discovered on every run.
|
|
41
|
+
|
|
42
|
+
It fixes what the model doesn't *know* (a convention, a gotcha), not what it
|
|
43
|
+
*can't do* — see the benchmark below.
|
|
44
|
+
|
|
45
|
+
## Quickstart (< 5 minutes)
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install errlore
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from errlore import AgentMemory
|
|
53
|
+
|
|
54
|
+
mem = AgentMemory("./agent_memory")
|
|
55
|
+
|
|
56
|
+
# 1. Agent failed -- record it
|
|
57
|
+
err_id = mem.log_error("gpt-5.5", "extraction", error="hallucinated dates")
|
|
58
|
+
|
|
59
|
+
# 2. You fixed it -- extract a lesson
|
|
60
|
+
mem.resolve(err_id, "Added date format validation",
|
|
61
|
+
lesson="For date extraction, demand ISO-8601 and verify against source")
|
|
62
|
+
|
|
63
|
+
# 3. Next similar task -- lessons + known issues injected automatically
|
|
64
|
+
inj = mem.inject_for("extract dates from contract", model="gpt-5.5",
|
|
65
|
+
task_type="extraction")
|
|
66
|
+
prompt = f"Your task: extract dates\n{inj.text}"
|
|
67
|
+
print(prompt)
|
|
68
|
+
|
|
69
|
+
# 4. Close the loop -- did the lesson help?
|
|
70
|
+
mem.report_outcome(inj, success=True)
|
|
71
|
+
|
|
72
|
+
# 5. Check stats
|
|
73
|
+
print(mem.stats())
|
|
74
|
+
# {'errors_total': 1, 'errors_resolved': 1, 'errors_unresolved': 0,
|
|
75
|
+
# 'lessons_total': 1, 'lessons_applied': 1, 'pending_injections': 0,
|
|
76
|
+
# 'trust': {'gpt-5.5': 0.5522...}}
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
No API keys needed. errlore itself never calls any LLM -- it manages local
|
|
80
|
+
JSONL files and does text matching. LLM calls are yours to make (or not).
|
|
81
|
+
|
|
82
|
+
## Does it actually reduce errors?
|
|
83
|
+
|
|
84
|
+
For the class of errors memory can fix — yes, and here's the honest version.
|
|
85
|
+
Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
|
|
86
|
+
(claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
|
|
87
|
+
Deterministic validators, no LLM judges; raw outputs committed in
|
|
88
|
+
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
|
|
89
|
+
you can recompute every number.
|
|
90
|
+
|
|
91
|
+
| arm | failures | fail rate |
|
|
92
|
+
|-----|----------|-----------|
|
|
93
|
+
| A: plain | 63/96 | 65.6% |
|
|
94
|
+
| B: with errlore | 20/96 | 20.8% |
|
|
95
|
+
|
|
96
|
+
Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
|
|
97
|
+
Split by error class:
|
|
98
|
+
|
|
99
|
+
- **Knowledge-gap errors** (workspace conventions: date formats, ID
|
|
100
|
+
normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
|
|
101
|
+
can't know a convention it was never told, so arm A fails almost by
|
|
102
|
+
construction; the result shows errlore **captures the fix once and re-supplies
|
|
103
|
+
it** on the next similar task, end to end. That store-and-inject loop is the
|
|
104
|
+
claim — not that memory teaches skills.
|
|
105
|
+
- **Capability-gap errors** (letter counting, string reversal): 17/48 ->
|
|
106
|
+
20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
|
|
107
|
+
model doesn't know, not what it can't do.
|
|
108
|
+
|
|
109
|
+
**Caveats, up front:** this is a single run at temperature 0 (LLM output is
|
|
110
|
+
still slightly non-deterministic, so exact fine-grained counts vary between
|
|
111
|
+
runs — the large knowledge-gap effect is robust; the capability-gap delta is
|
|
112
|
+
within noise). The knowledge-gap task families use conventions the model
|
|
113
|
+
demonstrably can't guess, which is the point — but it means the headline is
|
|
114
|
+
"the loop works," not "90% fewer errors everywhere."
|
|
115
|
+
|
|
116
|
+
Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
|
|
117
|
+
(needs an Anthropic API key; task families and validators ship in the repo).
|
|
118
|
+
|
|
119
|
+
## How it works
|
|
120
|
+
|
|
121
|
+
errlore runs three reinforcement loops around your agent:
|
|
122
|
+
|
|
123
|
+
### 1. Lesson loop
|
|
124
|
+
|
|
125
|
+
```
|
|
126
|
+
Agent fails --> log_error() --> resolve() + lesson
|
|
127
|
+
|
|
|
128
|
+
Agent runs <-- inject_for() <--------+
|
|
129
|
+
|
|
|
130
|
+
+--> report_outcome(success=True) --> lesson confidence +0.1
|
|
131
|
+
+--> report_outcome(success=False) --> lesson confidence -0.1
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Lessons with high confidence surface first. Unused lessons decay over time.
|
|
135
|
+
|
|
136
|
+
### 2. Known-issue loop
|
|
137
|
+
|
|
138
|
+
Per-model, per-task-type error tracking. When a model has failed on a task
|
|
139
|
+
type before, `inject_for` adds a warning block to the prompt. Separate from
|
|
140
|
+
lessons: lessons are *solutions*, known issues are *warnings*.
|
|
141
|
+
|
|
142
|
+
### 3. Trust loop *(experimental)*
|
|
143
|
+
|
|
144
|
+
Bayesian per-model weights with adaptive learning rate, cold-start blending,
|
|
145
|
+
entropy enforcement, and temporal decay. After enough observations, call
|
|
146
|
+
`mem.best_model("code_generation")` to pick the model that historically
|
|
147
|
+
performs best on that domain.
|
|
148
|
+
|
|
149
|
+
> **Status: experimental.** The engine is tested and works, but discrimination
|
|
150
|
+
> between models only emerges from a *spread* of real outcomes over time — feed
|
|
151
|
+
> it a stream that is mostly successes and every model converges near the cap.
|
|
152
|
+
> Treat `best_model()` as a hint to validate, not a proven router yet. The
|
|
153
|
+
> lesson + known-issue loops above are the proven core (see the A/B benchmark).
|
|
154
|
+
|
|
155
|
+
## Semantic retrieval (optional)
|
|
156
|
+
|
|
157
|
+
By default, errlore finds relevant lessons via word overlap (zero
|
|
158
|
+
dependencies). For higher recall on paraphrased queries, enable embedding
|
|
159
|
+
search:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
pip install errlore[embeddings] # installs fastembed + numpy
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
mem = AgentMemory("./agent_memory", embeddings=True)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
> The embedding model (~120 MB ONNX) is downloaded once on first use, then
|
|
170
|
+
> runs locally with no further network calls. The core (word-overlap) stays
|
|
171
|
+
> fully offline and dependency-free.
|
|
172
|
+
|
|
173
|
+
### Benchmark (adversarial paraphrasing)
|
|
174
|
+
|
|
175
|
+
Tested on 40 lessons with adversarially paraphrased queries
|
|
176
|
+
(`benchmarks/bench_retrieval.py`):
|
|
177
|
+
|
|
178
|
+
| Metric | word-overlap | embeddings |
|
|
179
|
+
|-----------|-------------|------------|
|
|
180
|
+
| recall@1 | 0.000 | 0.375 |
|
|
181
|
+
| recall@3 | 0.000 | 0.575 |
|
|
182
|
+
| recall@5 | 0.000 | 0.675 |
|
|
183
|
+
| MRR | 0.000 | 0.488 |
|
|
184
|
+
|
|
185
|
+
The gold set is intentionally adversarial (queries share few literal words
|
|
186
|
+
with the lesson text), which is why word-overlap scores zero. On natural
|
|
187
|
+
queries with shared vocabulary, word-overlap works fine.
|
|
188
|
+
|
|
189
|
+
## Integrations
|
|
190
|
+
|
|
191
|
+
errlore is framework-agnostic. It produces a text block; you put it in the
|
|
192
|
+
system prompt.
|
|
193
|
+
|
|
194
|
+
**Claude Code** — one command wires up failure-memory across sessions:
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
errlore init claude-code # or: --project for this repo only
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Failed Bash commands become lessons; every new session is briefed on past
|
|
201
|
+
pitfalls. See [examples/claude-code/](examples/claude-code/).
|
|
202
|
+
|
|
203
|
+
| Provider | Example |
|
|
204
|
+
|-------------|------------------------------------------------------|
|
|
205
|
+
| Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
|
|
206
|
+
| Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
|
|
207
|
+
| OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
|
|
208
|
+
| Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
|
|
209
|
+
| LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
|
|
210
|
+
|
|
211
|
+
The SDK examples run offline with `python examples/<name>.py` (mock responses,
|
|
212
|
+
no API keys). Set `use_api=True` to call real models.
|
|
213
|
+
|
|
214
|
+
### CLI
|
|
215
|
+
|
|
216
|
+
`pip install errlore` also installs an `errlore` command:
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
errlore init claude-code # install Claude Code hooks + settings
|
|
220
|
+
errlore stats # memory stats for a data dir (--data-dir)
|
|
221
|
+
errlore lessons # list stored lessons
|
|
222
|
+
errlore --version
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
## API overview
|
|
226
|
+
|
|
227
|
+
The main entry point is `AgentMemory`. All other classes are internal --
|
|
228
|
+
you only need them for advanced use.
|
|
229
|
+
|
|
230
|
+
| Method / Property | Description |
|
|
231
|
+
|-------------------------------|------------------------------------------------|
|
|
232
|
+
| `log_error(model, task_type, error)` | Record an error. Returns error ID. |
|
|
233
|
+
| `resolve(err_id, resolution, lesson)` | Mark error fixed, extract a lesson. |
|
|
234
|
+
| `inject_for(task, model)` | Build prompt injection (lessons + warnings). |
|
|
235
|
+
| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
|
|
236
|
+
| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
|
|
237
|
+
| `lessons(limit)` | List all lessons (sorted by confidence). |
|
|
238
|
+
| `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
|
|
239
|
+
| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
|
|
240
|
+
| `pending_injections()` | Injections not yet reported. |
|
|
241
|
+
| `stats()` | Aggregate counts + trust weights. |
|
|
242
|
+
| `.trust` | Access the underlying `TrustEngine` (or None). |
|
|
243
|
+
|
|
244
|
+
### Supporting classes (advanced)
|
|
245
|
+
|
|
246
|
+
| Class | Purpose |
|
|
247
|
+
|-------------------|--------------------------------------------|
|
|
248
|
+
| `LessonStore` | Low-level lesson CRUD + search. |
|
|
249
|
+
| `TrustEngine` | Bayesian trust weights with persistence. |
|
|
250
|
+
| `FeedbackSignal` | Typed quality signal for trust updates. |
|
|
251
|
+
| `Injection` | Dataclass returned by `inject_for`. |
|
|
252
|
+
|
|
253
|
+
## Data & privacy
|
|
254
|
+
|
|
255
|
+
- All data is stored in local JSONL files in the directory you specify.
|
|
256
|
+
- Nothing is sent to any server. errlore itself makes zero network calls.
|
|
257
|
+
- Works fully offline -- no API keys, no accounts, no telemetry.
|
|
258
|
+
- Files: `errors.jsonl`, `lessons.jsonl`, `injections.jsonl`, `trust.json`,
|
|
259
|
+
`model_accuracy.jsonl`.
|
|
260
|
+
- Sidecar files (auto-managed): `*.idx` (byte-offset index), `*.lock`
|
|
261
|
+
(filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
|
|
262
|
+
(embedding metadata), `trust.json` (trust engine state).
|
|
263
|
+
|
|
264
|
+
## Security
|
|
265
|
+
|
|
266
|
+
A lesson is **trusted prompt content by design** — it is injected into your
|
|
267
|
+
prompts and reaches the model. So:
|
|
268
|
+
|
|
269
|
+
- **Do not ingest lessons from untrusted sources without review.** Treat lesson
|
|
270
|
+
capture like a code review, not like user input. A malicious lesson is a
|
|
271
|
+
prompt-injection vector — and this is the real control, not the sanitizer.
|
|
272
|
+
- **What the sanitizer does (and does not) do.** The lesson *pattern* passes
|
|
273
|
+
`sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
|
|
274
|
+
so log blobs don't pollute the prompt. It is a noise filter, **not** an
|
|
275
|
+
injection defense — it does not neutralize natural-language instructions, and
|
|
276
|
+
the *solution* text is stored as you author it (so it can hold real code).
|
|
277
|
+
Don't rely on it to make untrusted lessons safe.
|
|
278
|
+
- You control what becomes a lesson (`resolve(..., lesson=...)` /
|
|
279
|
+
`add_lesson(...)`); nothing is auto-promoted from raw model output.
|
|
280
|
+
|
|
281
|
+
Report security issues to the address in [SECURITY.md](SECURITY.md).
|
|
282
|
+
|
|
283
|
+
## Scale & limits (honest)
|
|
284
|
+
|
|
285
|
+
errlore is built for **one process, thousands of lessons** — a single agent or
|
|
286
|
+
a coding-agent session, not a high-throughput fleet. Know the edges:
|
|
287
|
+
|
|
288
|
+
- **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
|
|
289
|
+
ledger each call, so at very high injection volumes it slows down (roughly
|
|
290
|
+
linear in total injections). Fine for interactive/agent use; log compaction
|
|
291
|
+
is the next roadmap item. If you don't need the reinforcement loop, you can
|
|
292
|
+
ignore `report_outcome` and the file stays small.
|
|
293
|
+
- **Single-process by default.** The lesson/error stores use cross-process file
|
|
294
|
+
locks and are safe to share, but the **trust engine and the optional vector
|
|
295
|
+
index are not cross-process safe** — two processes writing `trust.json` /
|
|
296
|
+
`vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
|
|
297
|
+
writer, or give each process its own `data_dir`. Multi-agent shared memory is
|
|
298
|
+
on the roadmap.
|
|
299
|
+
- **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
|
|
300
|
+
over a large existing lesson store is slow the first time (then incremental).
|
|
301
|
+
- Concurrency is tested across threads; **multi-process** stress is not yet in
|
|
302
|
+
the suite.
|
|
303
|
+
|
|
304
|
+
None of these bite at the scale errlore targets today; they're stated so you
|
|
305
|
+
can decide, not discover.
|
|
306
|
+
|
|
307
|
+
## Roadmap
|
|
308
|
+
|
|
309
|
+
- [ ] Log compaction for injections journal
|
|
310
|
+
- [ ] Async API (`alog_error`, `ainject_for`, etc.)
|
|
311
|
+
- [ ] Multi-agent shared memory (multiple agents, one lesson store)
|
|
312
|
+
- [ ] Lesson clustering and auto-summarization
|
|
313
|
+
- [ ] Dashboard / CLI for browsing lessons and trust weights
|
|
314
|
+
- [ ] Export/import for lesson sharing between projects
|
|
315
|
+
|
|
316
|
+
## License
|
|
317
|
+
|
|
318
|
+
MIT
|
|
@@ -16,15 +16,15 @@ runs end-to-end without network access.
|
|
|
16
16
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
+
import os
|
|
19
20
|
import tempfile
|
|
20
21
|
from pathlib import Path
|
|
21
22
|
|
|
22
23
|
from errlore import AgentMemory
|
|
23
24
|
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
|
|
27
|
-
MODEL = "claude-opus-4-8"
|
|
25
|
+
# Model is just a label to errlore (it never calls the API itself). Override
|
|
26
|
+
# with your own Claude model id; the default is a small, fast, cheap option.
|
|
27
|
+
MODEL = os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-latest")
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
# -- Agent wrapper -----------------------------------------------------------
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# errlore + Claude Code
|
|
2
|
+
|
|
3
|
+
Give your coding agent a memory of its own failures across sessions:
|
|
4
|
+
|
|
5
|
+
- **PostToolUse hook** — every failed Bash command is logged into errlore.
|
|
6
|
+
Resolve the ones you fixed (`mem.resolve(err_id, ..., lesson=...)`) or use
|
|
7
|
+
`mem.add_lesson()` to capture takeaways directly.
|
|
8
|
+
- **SessionStart hook** — each new session begins with a briefing block of
|
|
9
|
+
relevant lessons and per-tool KNOWN ISSUES, printed into the context.
|
|
10
|
+
|
|
11
|
+
## Setup (one command)
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install errlore
|
|
15
|
+
errlore init claude-code # global (~/.claude/settings.json)
|
|
16
|
+
errlore init claude-code --project # or this repo only (./.claude/settings.json)
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
That writes the two hook scripts (to `~/.errlore/hooks/`) and merges them into
|
|
20
|
+
your `settings.json` — idempotently, preserving any hooks you already have.
|
|
21
|
+
Restart Claude Code (or open a new session) to pick them up. Options:
|
|
22
|
+
`--data-dir` (where the memory lives, default `~/.errlore/claude-code`) and
|
|
23
|
+
`--hooks-dir`.
|
|
24
|
+
|
|
25
|
+
Handy afterwards: `errlore stats` and `errlore lessons` to see what it learned.
|
|
26
|
+
|
|
27
|
+
### Manual setup (if you'd rather wire it yourself)
|
|
28
|
+
|
|
29
|
+
The scripts in this folder (`errlore_posttooluse.py`, `errlore_sessionstart.py`)
|
|
30
|
+
plus `settings.json.example` show the shape: copy them somewhere stable, fix the
|
|
31
|
+
paths, and merge into `.claude/settings.json`. `export ERRLORE_DATA=...` picks
|
|
32
|
+
the memory dir.
|
|
33
|
+
|
|
34
|
+
Notes: hook event field names can differ between Claude Code versions —
|
|
35
|
+
the PostToolUse script reads them defensively and never breaks the agent
|
|
36
|
+
loop (exit 0 on anything unexpected). Check `claude --help` / the hooks
|
|
37
|
+
docs for your version if events don't arrive.
|
|
@@ -19,14 +19,15 @@ The ``if __name__`` block uses a mock instead of a real LLM call.
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
+
import os
|
|
22
23
|
import tempfile
|
|
23
24
|
from pathlib import Path
|
|
24
25
|
|
|
25
26
|
from errlore import AgentMemory, Injection
|
|
26
27
|
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
MODEL = "gpt-
|
|
28
|
+
# Model is just a label to errlore (it never calls the API itself). Override
|
|
29
|
+
# with your own; the default is a small, cheap, widely-available option.
|
|
30
|
+
MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
# -- errlore + LangChain glue ------------------------------------------------
|