errlore 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {errlore-0.1.2 → errlore-0.1.4}/CHANGELOG.md +39 -0
  2. {errlore-0.1.2 → errlore-0.1.4}/PKG-INFO +140 -12
  3. errlore-0.1.4/README.md +318 -0
  4. {errlore-0.1.2 → errlore-0.1.4}/examples/anthropic_agent.py +4 -4
  5. errlore-0.1.4/examples/claude-code/README.md +37 -0
  6. {errlore-0.1.2 → errlore-0.1.4}/examples/langchain_agent.py +4 -3
  7. {errlore-0.1.2 → errlore-0.1.4}/examples/openai_agent.py +4 -3
  8. {errlore-0.1.2 → errlore-0.1.4}/pyproject.toml +4 -1
  9. {errlore-0.1.2 → errlore-0.1.4}/site/index.html +41 -18
  10. {errlore-0.1.2 → errlore-0.1.4}/site/llms.txt +10 -6
  11. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/__init__.py +1 -1
  12. errlore-0.1.4/src/errlore/cli.py +200 -0
  13. errlore-0.1.4/src/errlore/integrations/__init__.py +5 -0
  14. errlore-0.1.4/src/errlore/integrations/claude_code.py +92 -0
  15. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/jsonl_writer.py +7 -8
  16. errlore-0.1.4/tests/test_cli.py +145 -0
  17. {errlore-0.1.2 → errlore-0.1.4}/tests/test_regressions.py +56 -0
  18. errlore-0.1.2/README.md +0 -190
  19. errlore-0.1.2/examples/claude-code/README.md +0 -23
  20. {errlore-0.1.2 → errlore-0.1.4}/.github/workflows/ci.yml +0 -0
  21. {errlore-0.1.2 → errlore-0.1.4}/.gitignore +0 -0
  22. {errlore-0.1.2 → errlore-0.1.4}/LICENSE +0 -0
  23. {errlore-0.1.2 → errlore-0.1.4}/SECURITY.md +0 -0
  24. {errlore-0.1.2 → errlore-0.1.4}/benchmarks/bench_error_reduction.py +0 -0
  25. {errlore-0.1.2 → errlore-0.1.4}/benchmarks/bench_retrieval.py +0 -0
  26. {errlore-0.1.2 → errlore-0.1.4}/benchmarks/results/error_reduction/report.md +0 -0
  27. {errlore-0.1.2 → errlore-0.1.4}/examples/claude-code/errlore_posttooluse.py +0 -0
  28. {errlore-0.1.2 → errlore-0.1.4}/examples/claude-code/errlore_sessionstart.py +0 -0
  29. {errlore-0.1.2 → errlore-0.1.4}/examples/claude-code/settings.json.example +0 -0
  30. {errlore-0.1.2 → errlore-0.1.4}/integrations/openwebui/README.md +0 -0
  31. {errlore-0.1.2 → errlore-0.1.4}/integrations/openwebui/errlore_feedback_action.py +0 -0
  32. {errlore-0.1.2 → errlore-0.1.4}/integrations/openwebui/errlore_memory_filter.py +0 -0
  33. {errlore-0.1.2 → errlore-0.1.4}/site/demo.gif +0 -0
  34. {errlore-0.1.2 → errlore-0.1.4}/site/demo_script.py +0 -0
  35. {errlore-0.1.2 → errlore-0.1.4}/site/og.png +0 -0
  36. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/__init__.py +0 -0
  37. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/classifier.py +0 -0
  38. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/injector.py +0 -0
  39. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/patterns.py +0 -0
  40. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/errmem/tracker.py +0 -0
  41. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/facade.py +0 -0
  42. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/__init__.py +0 -0
  43. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/jsonl_index.py +0 -0
  44. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/io/repair.py +0 -0
  45. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/lessons/__init__.py +0 -0
  46. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/lessons/models.py +0 -0
  47. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/lessons/store.py +0 -0
  48. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/py.typed +0 -0
  49. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/retrieval/__init__.py +0 -0
  50. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/retrieval/backend.py +0 -0
  51. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/retrieval/index.py +0 -0
  52. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/sanitize.py +0 -0
  53. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/trust/__init__.py +0 -0
  54. {errlore-0.1.2 → errlore-0.1.4}/src/errlore/trust/engine.py +0 -0
  55. {errlore-0.1.2 → errlore-0.1.4}/tests/conftest.py +0 -0
  56. {errlore-0.1.2 → errlore-0.1.4}/tests/test_errmem.py +0 -0
  57. {errlore-0.1.2 → errlore-0.1.4}/tests/test_facade.py +0 -0
  58. {errlore-0.1.2 → errlore-0.1.4}/tests/test_io.py +0 -0
  59. {errlore-0.1.2 → errlore-0.1.4}/tests/test_lessons.py +0 -0
  60. {errlore-0.1.2 → errlore-0.1.4}/tests/test_openwebui_integration.py +0 -0
  61. {errlore-0.1.2 → errlore-0.1.4}/tests/test_retrieval.py +0 -0
  62. {errlore-0.1.2 → errlore-0.1.4}/tests/test_trust.py +0 -0
@@ -5,6 +5,45 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [0.1.4] - 2026-07-09
9
+
10
+ ### Added
11
+ - **CLI** (`errlore` console command): `errlore init claude-code` writes the
12
+ two Claude Code hook scripts and idempotently merges them into your
13
+ `settings.json` (global or `--project`), preserving existing hooks — a
14
+ one-command install instead of copy/edit/merge. Plus `errlore stats` and
15
+ `errlore lessons`.
16
+ - `errlore.integrations.claude_code` — the hook logic (`post_tool_use`,
17
+ `session_start`) now ships in the package and is tested, so the generated
18
+ hooks are 3-line shims.
19
+
20
+ ### Changed
21
+ - README: honest A/B framing (the knowledge-gap baseline fails by
22
+ construction; the result shows the capture-and-re-supply loop works, not
23
+ that memory teaches skills; single-run-at-temp-0 caveat). Coding-agent-first
24
+ hero. Security section reworded — the sanitizer is a noise filter on the
25
+ pattern, not an injection defense. Added a "Scale & limits" section
26
+ (unbounded injections journal, single-process trust/vector index).
27
+ - Examples no longer hard-code aging frontier model ids; they use
28
+ `os.getenv(...)` with a small default.
29
+
30
+ ## [0.1.3] - 2026-07-06
31
+
32
+ ### Fixed
33
+ - Read-cache poisoning race in `JSONLWriter.read_all`: an append landing
34
+ mid-parse cached pre-append records under the post-append mtime/size, so
35
+ later reads returned stale data and `atomic_update` could silently drop
36
+ the concurrent record. Now caches only when the file is unchanged across
37
+ the parse (pre/post stat match). Found by pre-launch adversarial audit;
38
+ regression test added (Bug 3).
39
+ - Removed dead `JSONLWriter._invalidate_cache`.
40
+
41
+ ### Changed
42
+ - README: added the error-reduction A/B benchmark section (96 paired tasks,
43
+ 63 -> 20 failures, McNemar p=1.8e-09; knowledge-gap 46/48 -> 0/48,
44
+ capability-gap honestly worse at 17/48 -> 20/48), so the PyPI page shows
45
+ the headline evidence, not just the retrieval table.
46
+
8
47
  ## [0.1.2] - 2026-07-06
9
48
 
10
49
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: errlore
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server.
5
5
  Project-URL: Homepage, https://errlore.com
6
6
  Project-URL: Repository, https://github.com/Ma4etaSS/errlore
@@ -52,14 +52,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
52
52
  into the prompt for similar future tasks.
53
53
  - **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
54
54
  extraction tasks") injected as warnings.
55
- - **Trust** -- Bayesian per-model, per-domain trust weights: know which model to pick
56
- for which job, based on observed outcomes.
55
+ - **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
56
+ point for which model to pick per job, based on observed outcomes. Needs a spread of
57
+ real outcomes to separate models; shipped, but not yet proven on production traffic.
57
58
  - **Closed loop** -- errlore tracks whether an injected lesson actually helped and
58
59
  reinforces or decays it automatically.
59
60
 
60
61
  Embedded, file-based (JSONL), no server, no database, no API keys required.
61
62
  Works fully offline. Your data never leaves your machine.
62
63
 
64
+ ## Who it's for
65
+
66
+ errlore isn't memory for everything — it's memory for **failures**. It shines
67
+ wherever an agent repeats the *same class* of mistake:
68
+
69
+ - **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
70
+ the same bug or forgetting a project convention across sessions.
71
+ - **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
72
+ same date format, rounding rule, or schema field every week.
73
+ - **Any repeated-failure workflow** where a fix should stick the first time,
74
+ not be re-discovered on every run.
75
+
76
+ It fixes what the model doesn't *know* (a convention, a gotcha), not what it
77
+ *can't do* — see the benchmark below.
78
+
63
79
  ## Quickstart (< 5 minutes)
64
80
 
65
81
  ```bash
@@ -91,12 +107,49 @@ mem.report_outcome(inj, success=True)
91
107
  print(mem.stats())
92
108
  # {'errors_total': 1, 'errors_resolved': 1, 'errors_unresolved': 0,
93
109
  # 'lessons_total': 1, 'lessons_applied': 1, 'pending_injections': 0,
94
- # 'trust': {'gpt-5.5': 0.55}}
110
+ # 'trust': {'gpt-5.5': 0.5522...}}
95
111
  ```
96
112
 
97
113
  No API keys needed. errlore itself never calls any LLM -- it manages local
98
114
  JSONL files and does text matching. LLM calls are yours to make (or not).
99
115
 
116
+ ## Does it actually reduce errors?
117
+
118
+ For the class of errors memory can fix — yes, and here's the honest version.
119
+ Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
120
+ (claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
121
+ Deterministic validators, no LLM judges; raw outputs committed in
122
+ [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
123
+ you can recompute every number.
124
+
125
+ | arm | failures | fail rate |
126
+ |-----|----------|-----------|
127
+ | A: plain | 63/96 | 65.6% |
128
+ | B: with errlore | 20/96 | 20.8% |
129
+
130
+ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
131
+ Split by error class:
132
+
133
+ - **Knowledge-gap errors** (workspace conventions: date formats, ID
134
+ normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
135
+ can't know a convention it was never told, so arm A fails almost by
136
+ construction; the result shows errlore **captures the fix once and re-supplies
137
+ it** on the next similar task, end to end. That store-and-inject loop is the
138
+ claim — not that memory teaches skills.
139
+ - **Capability-gap errors** (letter counting, string reversal): 17/48 ->
140
+ 20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
141
+ model doesn't know, not what it can't do.
142
+
143
+ **Caveats, up front:** this is a single run at temperature 0 (LLM output is
144
+ still slightly non-deterministic, so exact fine-grained counts vary between
145
+ runs — the large knowledge-gap effect is robust; the capability-gap delta is
146
+ within noise). The knowledge-gap task families use conventions the model
147
+ demonstrably can't guess, which is the point — but it means the headline is
148
+ "the loop works," not "90% fewer errors everywhere."
149
+
150
+ Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
151
+ (needs an Anthropic API key; task families and validators ship in the repo).
152
+
100
153
  ## How it works
101
154
 
102
155
  errlore runs three reinforcement loops around your agent:
@@ -120,13 +173,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
120
173
  type before, `inject_for` adds a warning block to the prompt. Separate from
121
174
  lessons: lessons are *solutions*, known issues are *warnings*.
122
175
 
123
- ### 3. Trust loop
176
+ ### 3. Trust loop *(experimental)*
124
177
 
125
178
  Bayesian per-model weights with adaptive learning rate, cold-start blending,
126
179
  entropy enforcement, and temporal decay. After enough observations, call
127
180
  `mem.best_model("code_generation")` to pick the model that historically
128
181
  performs best on that domain.
129
182
 
183
+ > **Status: experimental.** The engine is tested and works, but discrimination
184
+ > between models only emerges from a *spread* of real outcomes over time — feed
185
+ > it a stream that is mostly successes and every model converges near the cap.
186
+ > Treat `best_model()` as a hint to validate, not a proven router yet. The
187
+ > lesson + known-issue loops above are the proven core (see the A/B benchmark).
188
+
130
189
  ## Semantic retrieval (optional)
131
190
 
132
191
  By default, errlore finds relevant lessons via word overlap (zero
@@ -141,6 +200,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
141
200
  mem = AgentMemory("./agent_memory", embeddings=True)
142
201
  ```
143
202
 
203
+ > The embedding model (~120 MB ONNX) is downloaded once on first use, then
204
+ > runs locally with no further network calls. The core (word-overlap) stays
205
+ > fully offline and dependency-free.
206
+
144
207
  ### Benchmark (adversarial paraphrasing)
145
208
 
146
209
  Tested on 40 lessons with adversarially paraphrased queries
@@ -162,15 +225,37 @@ queries with shared vocabulary, word-overlap works fine.
162
225
  errlore is framework-agnostic. It produces a text block; you put it in the
163
226
  system prompt.
164
227
 
165
- | Provider | Example |
166
- |------------|------------------------------------------------------|
167
- | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
168
- | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
169
- | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
228
+ **Claude Code** — one command wires up failure-memory across sessions:
229
+
230
+ ```bash
231
+ errlore init claude-code # or: --project for this repo only
232
+ ```
233
+
234
+ Failed Bash commands become lessons; every new session is briefed on past
235
+ pitfalls. See [examples/claude-code/](examples/claude-code/).
236
+
237
+ | Provider | Example |
238
+ |-------------|------------------------------------------------------|
239
+ | Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
240
+ | Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
241
+ | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
242
+ | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
243
+ | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
170
244
 
171
- All examples run offline with `python examples/<name>.py` (mock responses,
245
+ The SDK examples run offline with `python examples/<name>.py` (mock responses,
172
246
  no API keys). Set `use_api=True` to call real models.
173
247
 
248
+ ### CLI
249
+
250
+ `pip install errlore` also installs an `errlore` command:
251
+
252
+ ```bash
253
+ errlore init claude-code # install Claude Code hooks + settings
254
+ errlore stats # memory stats for a data dir (--data-dir)
255
+ errlore lessons # list stored lessons
256
+ errlore --version
257
+ ```
258
+
174
259
  ## API overview
175
260
 
176
261
  The main entry point is `AgentMemory`. All other classes are internal --
@@ -184,7 +269,7 @@ you only need them for advanced use.
184
269
  | `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
185
270
  | `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
186
271
  | `lessons(limit)` | List all lessons (sorted by confidence). |
187
- | `best_model(domain)` | Model with the highest trust weight. |
272
+ | `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
188
273
  | `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
189
274
  | `pending_injections()` | Injections not yet reported. |
190
275
  | `stats()` | Aggregate counts + trust weights. |
@@ -210,6 +295,49 @@ you only need them for advanced use.
210
295
  (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
211
296
  (embedding metadata), `trust.json` (trust engine state).
212
297
 
298
+ ## Security
299
+
300
+ A lesson is **trusted prompt content by design** — it is injected into your
301
+ prompts and reaches the model. So:
302
+
303
+ - **Do not ingest lessons from untrusted sources without review.** Treat lesson
304
+ capture like a code review, not like user input. A malicious lesson is a
305
+ prompt-injection vector — and this is the real control, not the sanitizer.
306
+ - **What the sanitizer does (and does not) do.** The lesson *pattern* passes
307
+ `sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
308
+ so log blobs don't pollute the prompt. It is a noise filter, **not** an
309
+ injection defense — it does not neutralize natural-language instructions, and
310
+ the *solution* text is stored as you author it (so it can hold real code).
311
+ Don't rely on it to make untrusted lessons safe.
312
+ - You control what becomes a lesson (`resolve(..., lesson=...)` /
313
+ `add_lesson(...)`); nothing is auto-promoted from raw model output.
314
+
315
+ Report security issues to the address in [SECURITY.md](SECURITY.md).
316
+
317
+ ## Scale & limits (honest)
318
+
319
+ errlore is built for **one process, thousands of lessons** — a single agent or
320
+ a coding-agent session, not a high-throughput fleet. Know the edges:
321
+
322
+ - **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
323
+ ledger each call, so at very high injection volumes it slows down (roughly
324
+ linear in total injections). Fine for interactive/agent use; log compaction
325
+ is the next roadmap item. If you don't need the reinforcement loop, you can
326
+ ignore `report_outcome` and the file stays small.
327
+ - **Single-process by default.** The lesson/error stores use cross-process file
328
+ locks and are safe to share, but the **trust engine and the optional vector
329
+ index are not cross-process safe** — two processes writing `trust.json` /
330
+ `vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
331
+ writer, or give each process its own `data_dir`. Multi-agent shared memory is
332
+ on the roadmap.
333
+ - **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
334
+ over a large existing lesson store is slow the first time (then incremental).
335
+ - Concurrency is tested across threads; **multi-process** stress is not yet in
336
+ the suite.
337
+
338
+ None of these bite at the scale errlore targets today; they're stated so you
339
+ can decide, not discover.
340
+
213
341
  ## Roadmap
214
342
 
215
343
  - [ ] Log compaction for injections journal
@@ -0,0 +1,318 @@
1
+ # errlore
2
+
3
+ **Memory for AI agents that learns from failures.**
4
+
5
+ *Stop the second mistake, not just the first.*
6
+
7
+ ![errlore demo: monday failure becomes a lesson, tuesday's prompt gets it injected](https://errlore.com/demo.gif)
8
+
9
+ [![CI](https://github.com/Ma4etaSS/errlore/actions/workflows/ci.yml/badge.svg)](https://github.com/Ma4etaSS/errlore/actions/workflows/ci.yml)
10
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://python.org)
11
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
12
+
13
+ Extracted from a 324K LOC production multi-LLM orchestration system, keeping the one part that demonstrably worked: the error-memory loop that made agents stop repeating mistakes.
14
+
15
+ Your agent keeps making the same mistakes. errlore fixes that:
16
+
17
+ - **Lessons** -- every resolved failure becomes a lesson; relevant lessons are injected
18
+ into the prompt for similar future tasks.
19
+ - **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
20
+ extraction tasks") injected as warnings.
21
+ - **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
22
+ point for which model to pick per job, based on observed outcomes. Needs a spread of
23
+ real outcomes to separate models; shipped, but not yet proven on production traffic.
24
+ - **Closed loop** -- errlore tracks whether an injected lesson actually helped and
25
+ reinforces or decays it automatically.
26
+
27
+ Embedded, file-based (JSONL), no server, no database, no API keys required.
28
+ Works fully offline. Your data never leaves your machine.
29
+
30
+ ## Who it's for
31
+
32
+ errlore isn't memory for everything — it's memory for **failures**. It shines
33
+ wherever an agent repeats the *same class* of mistake:
34
+
35
+ - **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
36
+ the same bug or forgetting a project convention across sessions.
37
+ - **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
38
+ same date format, rounding rule, or schema field every week.
39
+ - **Any repeated-failure workflow** where a fix should stick the first time,
40
+ not be re-discovered on every run.
41
+
42
+ It fixes what the model doesn't *know* (a convention, a gotcha), not what it
43
+ *can't do* — see the benchmark below.
44
+
45
+ ## Quickstart (< 5 minutes)
46
+
47
+ ```bash
48
+ pip install errlore
49
+ ```
50
+
51
+ ```python
52
+ from errlore import AgentMemory
53
+
54
+ mem = AgentMemory("./agent_memory")
55
+
56
+ # 1. Agent failed -- record it
57
+ err_id = mem.log_error("gpt-5.5", "extraction", error="hallucinated dates")
58
+
59
+ # 2. You fixed it -- extract a lesson
60
+ mem.resolve(err_id, "Added date format validation",
61
+ lesson="For date extraction, demand ISO-8601 and verify against source")
62
+
63
+ # 3. Next similar task -- lessons + known issues injected automatically
64
+ inj = mem.inject_for("extract dates from contract", model="gpt-5.5",
65
+ task_type="extraction")
66
+ prompt = f"Your task: extract dates\n{inj.text}"
67
+ print(prompt)
68
+
69
+ # 4. Close the loop -- did the lesson help?
70
+ mem.report_outcome(inj, success=True)
71
+
72
+ # 5. Check stats
73
+ print(mem.stats())
74
+ # {'errors_total': 1, 'errors_resolved': 1, 'errors_unresolved': 0,
75
+ # 'lessons_total': 1, 'lessons_applied': 1, 'pending_injections': 0,
76
+ # 'trust': {'gpt-5.5': 0.5522...}}
77
+ ```
78
+
79
+ No API keys needed. errlore itself never calls any LLM -- it manages local
80
+ JSONL files and does text matching. LLM calls are yours to make (or not).
81
+
82
+ ## Does it actually reduce errors?
83
+
84
+ For the class of errors memory can fix — yes, and here's the honest version.
85
+ Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
86
+ (claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
87
+ Deterministic validators, no LLM judges; raw outputs committed in
88
+ [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
89
+ you can recompute every number.
90
+
91
+ | arm | failures | fail rate |
92
+ |-----|----------|-----------|
93
+ | A: plain | 63/96 | 65.6% |
94
+ | B: with errlore | 20/96 | 20.8% |
95
+
96
+ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
97
+ Split by error class:
98
+
99
+ - **Knowledge-gap errors** (workspace conventions: date formats, ID
100
+ normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
101
+ can't know a convention it was never told, so arm A fails almost by
102
+ construction; the result shows errlore **captures the fix once and re-supplies
103
+ it** on the next similar task, end to end. That store-and-inject loop is the
104
+ claim — not that memory teaches skills.
105
+ - **Capability-gap errors** (letter counting, string reversal): 17/48 ->
106
+ 20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
107
+ model doesn't know, not what it can't do.
108
+
109
+ **Caveats, up front:** this is a single run at temperature 0 (LLM output is
110
+ still slightly non-deterministic, so exact fine-grained counts vary between
111
+ runs — the large knowledge-gap effect is robust; the capability-gap delta is
112
+ within noise). The knowledge-gap task families use conventions the model
113
+ demonstrably can't guess, which is the point — but it means the headline is
114
+ "the loop works," not "90% fewer errors everywhere."
115
+
116
+ Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
117
+ (needs an Anthropic API key; task families and validators ship in the repo).
118
+
119
+ ## How it works
120
+
121
+ errlore runs three reinforcement loops around your agent:
122
+
123
+ ### 1. Lesson loop
124
+
125
+ ```
126
+ Agent fails --> log_error() --> resolve() + lesson
127
+ |
128
+ Agent runs <-- inject_for() <--------+
129
+ |
130
+ +--> report_outcome(success=True) --> lesson confidence +0.1
131
+ +--> report_outcome(success=False) --> lesson confidence -0.1
132
+ ```
133
+
134
+ Lessons with high confidence surface first. Unused lessons decay over time.
135
+
136
+ ### 2. Known-issue loop
137
+
138
+ Per-model, per-task-type error tracking. When a model has failed on a task
139
+ type before, `inject_for` adds a warning block to the prompt. Separate from
140
+ lessons: lessons are *solutions*, known issues are *warnings*.
141
+
142
+ ### 3. Trust loop *(experimental)*
143
+
144
+ Bayesian per-model weights with adaptive learning rate, cold-start blending,
145
+ entropy enforcement, and temporal decay. After enough observations, call
146
+ `mem.best_model("code_generation")` to pick the model that historically
147
+ performs best on that domain.
148
+
149
+ > **Status: experimental.** The engine is tested and works, but discrimination
150
+ > between models only emerges from a *spread* of real outcomes over time — feed
151
+ > it a stream that is mostly successes and every model converges near the cap.
152
+ > Treat `best_model()` as a hint to validate, not a proven router yet. The
153
+ > lesson + known-issue loops above are the proven core (see the A/B benchmark).
154
+
155
+ ## Semantic retrieval (optional)
156
+
157
+ By default, errlore finds relevant lessons via word overlap (zero
158
+ dependencies). For higher recall on paraphrased queries, enable embedding
159
+ search:
160
+
161
+ ```bash
162
+ pip install errlore[embeddings] # installs fastembed + numpy
163
+ ```
164
+
165
+ ```python
166
+ mem = AgentMemory("./agent_memory", embeddings=True)
167
+ ```
168
+
169
+ > The embedding model (~120 MB ONNX) is downloaded once on first use, then
170
+ > runs locally with no further network calls. The core (word-overlap) stays
171
+ > fully offline and dependency-free.
172
+
173
+ ### Benchmark (adversarial paraphrasing)
174
+
175
+ Tested on 40 lessons with adversarially paraphrased queries
176
+ (`benchmarks/bench_retrieval.py`):
177
+
178
+ | Metric | word-overlap | embeddings |
179
+ |-----------|-------------|------------|
180
+ | recall@1 | 0.000 | 0.375 |
181
+ | recall@3 | 0.000 | 0.575 |
182
+ | recall@5 | 0.000 | 0.675 |
183
+ | MRR | 0.000 | 0.488 |
184
+
185
+ The gold set is intentionally adversarial (queries share few literal words
186
+ with the lesson text), which is why word-overlap scores zero. On natural
187
+ queries with shared vocabulary, word-overlap works fine.
188
+
189
+ ## Integrations
190
+
191
+ errlore is framework-agnostic. It produces a text block; you put it in the
192
+ system prompt.
193
+
194
+ **Claude Code** — one command wires up failure-memory across sessions:
195
+
196
+ ```bash
197
+ errlore init claude-code # or: --project for this repo only
198
+ ```
199
+
200
+ Failed Bash commands become lessons; every new session is briefed on past
201
+ pitfalls. See [examples/claude-code/](examples/claude-code/).
202
+
203
+ | Provider | Example |
204
+ |-------------|------------------------------------------------------|
205
+ | Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
206
+ | Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
207
+ | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
208
+ | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
209
+ | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
210
+
211
+ The SDK examples run offline with `python examples/<name>.py` (mock responses,
212
+ no API keys). Set `use_api=True` to call real models.
213
+
214
+ ### CLI
215
+
216
+ `pip install errlore` also installs an `errlore` command:
217
+
218
+ ```bash
219
+ errlore init claude-code # install Claude Code hooks + settings
220
+ errlore stats # memory stats for a data dir (--data-dir)
221
+ errlore lessons # list stored lessons
222
+ errlore --version
223
+ ```
224
+
225
+ ## API overview
226
+
227
+ The main entry point is `AgentMemory`. All other classes are internal --
228
+ you only need them for advanced use.
229
+
230
+ | Method / Property | Description |
231
+ |-------------------------------|------------------------------------------------|
232
+ | `log_error(model, task_type, error)` | Record an error. Returns error ID. |
233
+ | `resolve(err_id, resolution, lesson)` | Mark error fixed, extract a lesson. |
234
+ | `inject_for(task, model)` | Build prompt injection (lessons + warnings). |
235
+ | `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
236
+ | `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
237
+ | `lessons(limit)` | List all lessons (sorted by confidence). |
238
+ | `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
239
+ | `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
240
+ | `pending_injections()` | Injections not yet reported. |
241
+ | `stats()` | Aggregate counts + trust weights. |
242
+ | `.trust` | Access the underlying `TrustEngine` (or None). |
243
+
244
+ ### Supporting classes (advanced)
245
+
246
+ | Class | Purpose |
247
+ |-------------------|--------------------------------------------|
248
+ | `LessonStore` | Low-level lesson CRUD + search. |
249
+ | `TrustEngine` | Bayesian trust weights with persistence. |
250
+ | `FeedbackSignal` | Typed quality signal for trust updates. |
251
+ | `Injection` | Dataclass returned by `inject_for`. |
252
+
253
+ ## Data & privacy
254
+
255
+ - All data is stored in local JSONL files in the directory you specify.
256
+ - Nothing is sent to any server. errlore itself makes zero network calls.
257
+ - Works fully offline -- no API keys, no accounts, no telemetry.
258
+ - Files: `errors.jsonl`, `lessons.jsonl`, `injections.jsonl`, `trust.json`,
259
+ `model_accuracy.jsonl`.
260
+ - Sidecar files (auto-managed): `*.idx` (byte-offset index), `*.lock`
261
+ (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
262
+ (embedding metadata), `trust.json` (trust engine state).
263
+
264
+ ## Security
265
+
266
+ A lesson is **trusted prompt content by design** — it is injected into your
267
+ prompts and reaches the model. So:
268
+
269
+ - **Do not ingest lessons from untrusted sources without review.** Treat lesson
270
+ capture like a code review, not like user input. A malicious lesson is a
271
+ prompt-injection vector — and this is the real control, not the sanitizer.
272
+ - **What the sanitizer does (and does not) do.** The lesson *pattern* passes
273
+ `sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
274
+ so log blobs don't pollute the prompt. It is a noise filter, **not** an
275
+ injection defense — it does not neutralize natural-language instructions, and
276
+ the *solution* text is stored as you author it (so it can hold real code).
277
+ Don't rely on it to make untrusted lessons safe.
278
+ - You control what becomes a lesson (`resolve(..., lesson=...)` /
279
+ `add_lesson(...)`); nothing is auto-promoted from raw model output.
280
+
281
+ Report security issues to the address in [SECURITY.md](SECURITY.md).
282
+
283
+ ## Scale & limits (honest)
284
+
285
+ errlore is built for **one process, thousands of lessons** — a single agent or
286
+ a coding-agent session, not a high-throughput fleet. Know the edges:
287
+
288
+ - **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
289
+ ledger each call, so at very high injection volumes it slows down (roughly
290
+ linear in total injections). Fine for interactive/agent use; log compaction
291
+ is the next roadmap item. If you don't need the reinforcement loop, you can
292
+ ignore `report_outcome` and the file stays small.
293
+ - **Single-process by default.** The lesson/error stores use cross-process file
294
+ locks and are safe to share, but the **trust engine and the optional vector
295
+ index are not cross-process safe** — two processes writing `trust.json` /
296
+ `vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
297
+ writer, or give each process its own `data_dir`. Multi-agent shared memory is
298
+ on the roadmap.
299
+ - **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
300
+ over a large existing lesson store is slow the first time (then incremental).
301
+ - Concurrency is tested across threads; **multi-process** stress is not yet in
302
+ the suite.
303
+
304
+ None of these bite at the scale errlore targets today; they're stated so you
305
+ can decide, not discover.
306
+
307
+ ## Roadmap
308
+
309
+ - [ ] Log compaction for injections journal
310
+ - [ ] Async API (`alog_error`, `ainject_for`, etc.)
311
+ - [ ] Multi-agent shared memory (multiple agents, one lesson store)
312
+ - [ ] Lesson clustering and auto-summarization
313
+ - [ ] Dashboard / CLI for browsing lessons and trust weights
314
+ - [ ] Export/import for lesson sharing between projects
315
+
316
+ ## License
317
+
318
+ MIT
@@ -16,15 +16,15 @@ runs end-to-end without network access.
16
16
 
17
17
  from __future__ import annotations
18
18
 
19
+ import os
19
20
  import tempfile
20
21
  from pathlib import Path
21
22
 
22
23
  from errlore import AgentMemory
23
24
 
24
- # Current Claude lineup (2026): claude-fable-5 (most capable),
25
- # claude-opus-4-8 (recommended default), claude-sonnet-4-6 (speed/cost balance),
26
- # claude-haiku-4-5 (fastest). Use exact IDs as-is -- no date suffixes.
27
- MODEL = "claude-opus-4-8"
25
+ # Model is just a label to errlore (it never calls the API itself). Override
26
+ # with your own Claude model id; the default is a small, fast, cheap option.
27
+ MODEL = os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-latest")
28
28
 
29
29
 
30
30
  # -- Agent wrapper -----------------------------------------------------------
@@ -0,0 +1,37 @@
1
+ # errlore + Claude Code
2
+
3
+ Give your coding agent a memory of its own failures across sessions:
4
+
5
+ - **PostToolUse hook** — every failed Bash command is logged into errlore.
6
+ Resolve the ones you fixed (`mem.resolve(err_id, ..., lesson=...)`) or use
7
+ `mem.add_lesson()` to capture takeaways directly.
8
+ - **SessionStart hook** — each new session begins with a briefing block of
9
+ relevant lessons and per-tool KNOWN ISSUES, printed into the context.
10
+
11
+ ## Setup (one command)
12
+
13
+ ```bash
14
+ pip install errlore
15
+ errlore init claude-code # global (~/.claude/settings.json)
16
+ errlore init claude-code --project # or this repo only (./.claude/settings.json)
17
+ ```
18
+
19
+ That writes the two hook scripts (to `~/.errlore/hooks/`) and merges them into
20
+ your `settings.json` — idempotently, preserving any hooks you already have.
21
+ Restart Claude Code (or open a new session) to pick them up. Options:
22
+ `--data-dir` (where the memory lives, default `~/.errlore/claude-code`) and
23
+ `--hooks-dir`.
24
+
25
+ Handy afterwards: `errlore stats` and `errlore lessons` to see what it learned.
26
+
27
+ ### Manual setup (if you'd rather wire it yourself)
28
+
29
+ The scripts in this folder (`errlore_posttooluse.py`, `errlore_sessionstart.py`)
30
+ plus `settings.json.example` show the shape: copy them somewhere stable, fix the
31
+ paths, and merge into `.claude/settings.json`. `export ERRLORE_DATA=...` picks
32
+ the memory dir.
33
+
34
+ Notes: hook event field names can differ between Claude Code versions —
35
+ the PostToolUse script reads them defensively and never breaks the agent
36
+ loop (exit 0 on anything unexpected). Check `claude --help` / the hooks
37
+ docs for your version if events don't arrive.
@@ -19,14 +19,15 @@ The ``if __name__`` block uses a mock instead of a real LLM call.
19
19
 
20
20
  from __future__ import annotations
21
21
 
22
+ import os
22
23
  import tempfile
23
24
  from pathlib import Path
24
25
 
25
26
  from errlore import AgentMemory, Injection
26
27
 
27
- # Current OpenAI lineup (2026): gpt-5.5 (frontier, recommended),
28
- # gpt-5.4-mini / gpt-5.4-nano (lower latency & cost).
29
- MODEL = "gpt-5.5"
28
+ # Model is just a label to errlore (it never calls the API itself). Override
29
+ # with your own; the default is a small, cheap, widely-available option.
30
+ MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
30
31
 
31
32
 
32
33
  # -- errlore + LangChain glue ------------------------------------------------