errlore 0.1.3__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {errlore-0.1.3 → errlore-0.1.4}/CHANGELOG.md +22 -0
  2. {errlore-0.1.3 → errlore-0.1.4}/PKG-INFO +122 -20
  3. {errlore-0.1.3 → errlore-0.1.4}/README.md +121 -19
  4. {errlore-0.1.3 → errlore-0.1.4}/examples/anthropic_agent.py +4 -4
  5. errlore-0.1.4/examples/claude-code/README.md +37 -0
  6. {errlore-0.1.3 → errlore-0.1.4}/examples/langchain_agent.py +4 -3
  7. {errlore-0.1.3 → errlore-0.1.4}/examples/openai_agent.py +4 -3
  8. {errlore-0.1.3 → errlore-0.1.4}/pyproject.toml +4 -1
  9. {errlore-0.1.3 → errlore-0.1.4}/site/index.html +22 -19
  10. {errlore-0.1.3 → errlore-0.1.4}/site/llms.txt +9 -6
  11. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/__init__.py +1 -1
  12. errlore-0.1.4/src/errlore/cli.py +200 -0
  13. errlore-0.1.4/src/errlore/integrations/__init__.py +5 -0
  14. errlore-0.1.4/src/errlore/integrations/claude_code.py +92 -0
  15. errlore-0.1.4/tests/test_cli.py +145 -0
  16. errlore-0.1.3/examples/claude-code/README.md +0 -23
  17. {errlore-0.1.3 → errlore-0.1.4}/.github/workflows/ci.yml +0 -0
  18. {errlore-0.1.3 → errlore-0.1.4}/.gitignore +0 -0
  19. {errlore-0.1.3 → errlore-0.1.4}/LICENSE +0 -0
  20. {errlore-0.1.3 → errlore-0.1.4}/SECURITY.md +0 -0
  21. {errlore-0.1.3 → errlore-0.1.4}/benchmarks/bench_error_reduction.py +0 -0
  22. {errlore-0.1.3 → errlore-0.1.4}/benchmarks/bench_retrieval.py +0 -0
  23. {errlore-0.1.3 → errlore-0.1.4}/benchmarks/results/error_reduction/report.md +0 -0
  24. {errlore-0.1.3 → errlore-0.1.4}/examples/claude-code/errlore_posttooluse.py +0 -0
  25. {errlore-0.1.3 → errlore-0.1.4}/examples/claude-code/errlore_sessionstart.py +0 -0
  26. {errlore-0.1.3 → errlore-0.1.4}/examples/claude-code/settings.json.example +0 -0
  27. {errlore-0.1.3 → errlore-0.1.4}/integrations/openwebui/README.md +0 -0
  28. {errlore-0.1.3 → errlore-0.1.4}/integrations/openwebui/errlore_feedback_action.py +0 -0
  29. {errlore-0.1.3 → errlore-0.1.4}/integrations/openwebui/errlore_memory_filter.py +0 -0
  30. {errlore-0.1.3 → errlore-0.1.4}/site/demo.gif +0 -0
  31. {errlore-0.1.3 → errlore-0.1.4}/site/demo_script.py +0 -0
  32. {errlore-0.1.3 → errlore-0.1.4}/site/og.png +0 -0
  33. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/__init__.py +0 -0
  34. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/classifier.py +0 -0
  35. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/injector.py +0 -0
  36. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/patterns.py +0 -0
  37. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/errmem/tracker.py +0 -0
  38. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/facade.py +0 -0
  39. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/__init__.py +0 -0
  40. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/jsonl_index.py +0 -0
  41. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/jsonl_writer.py +0 -0
  42. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/io/repair.py +0 -0
  43. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/lessons/__init__.py +0 -0
  44. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/lessons/models.py +0 -0
  45. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/lessons/store.py +0 -0
  46. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/py.typed +0 -0
  47. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/retrieval/__init__.py +0 -0
  48. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/retrieval/backend.py +0 -0
  49. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/retrieval/index.py +0 -0
  50. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/sanitize.py +0 -0
  51. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/trust/__init__.py +0 -0
  52. {errlore-0.1.3 → errlore-0.1.4}/src/errlore/trust/engine.py +0 -0
  53. {errlore-0.1.3 → errlore-0.1.4}/tests/conftest.py +0 -0
  54. {errlore-0.1.3 → errlore-0.1.4}/tests/test_errmem.py +0 -0
  55. {errlore-0.1.3 → errlore-0.1.4}/tests/test_facade.py +0 -0
  56. {errlore-0.1.3 → errlore-0.1.4}/tests/test_io.py +0 -0
  57. {errlore-0.1.3 → errlore-0.1.4}/tests/test_lessons.py +0 -0
  58. {errlore-0.1.3 → errlore-0.1.4}/tests/test_openwebui_integration.py +0 -0
  59. {errlore-0.1.3 → errlore-0.1.4}/tests/test_regressions.py +0 -0
  60. {errlore-0.1.3 → errlore-0.1.4}/tests/test_retrieval.py +0 -0
  61. {errlore-0.1.3 → errlore-0.1.4}/tests/test_trust.py +0 -0
@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [0.1.4] - 2026-07-09
9
+
10
+ ### Added
11
+ - **CLI** (`errlore` console command): `errlore init claude-code` writes the
12
+ two Claude Code hook scripts and idempotently merges them into your
13
+ `settings.json` (global or `--project`), preserving existing hooks — a
14
+ one-command install instead of copy/edit/merge. Plus `errlore stats` and
15
+ `errlore lessons`.
16
+ - `errlore.integrations.claude_code` — the hook logic (`post_tool_use`,
17
+ `session_start`) now ships in the package and is tested, so the generated
18
+ hooks are 3-line shims.
19
+
20
+ ### Changed
21
+ - README: honest A/B framing (the knowledge-gap baseline fails by
22
+ construction; the result shows the capture-and-re-supply loop works, not
23
+ that memory teaches skills; single-run-at-temp-0 caveat). Coding-agent-first
24
+ hero. Security section reworded — the sanitizer is a noise filter on the
25
+ pattern, not an injection defense. Added a "Scale & limits" section
26
+ (unbounded injections journal, single-process trust/vector index).
27
+ - Examples no longer hard-code aging frontier model ids; they use
28
+ `os.getenv(...)` with a small default.
29
+
8
30
  ## [0.1.3] - 2026-07-06
9
31
 
10
32
  ### Fixed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: errlore
3
- Version: 0.1.3
3
+ Version: 0.1.4
4
4
  Summary: Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server.
5
5
  Project-URL: Homepage, https://errlore.com
6
6
  Project-URL: Repository, https://github.com/Ma4etaSS/errlore
@@ -52,14 +52,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
52
52
  into the prompt for similar future tasks.
53
53
  - **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
54
54
  extraction tasks") injected as warnings.
55
- - **Trust** -- Bayesian per-model, per-domain trust weights: know which model to pick
56
- for which job, based on observed outcomes.
55
+ - **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
56
+ point for which model to pick per job, based on observed outcomes. Needs a spread of
57
+ real outcomes to separate models; shipped, but not yet proven on production traffic.
57
58
  - **Closed loop** -- errlore tracks whether an injected lesson actually helped and
58
59
  reinforces or decays it automatically.
59
60
 
60
61
  Embedded, file-based (JSONL), no server, no database, no API keys required.
61
62
  Works fully offline. Your data never leaves your machine.
62
63
 
64
+ ## Who it's for
65
+
66
+ errlore isn't memory for everything — it's memory for **failures**. It shines
67
+ wherever an agent repeats the *same class* of mistake:
68
+
69
+ - **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
70
+ the same bug or forgetting a project convention across sessions.
71
+ - **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
72
+ same date format, rounding rule, or schema field every week.
73
+ - **Any repeated-failure workflow** where a fix should stick the first time,
74
+ not be re-discovered on every run.
75
+
76
+ It fixes what the model doesn't *know* (a convention, a gotcha), not what it
77
+ *can't do* — see the benchmark below.
78
+
63
79
  ## Quickstart (< 5 minutes)
64
80
 
65
81
  ```bash
@@ -99,11 +115,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
99
115
 
100
116
  ## Does it actually reduce errors?
101
117
 
102
- Yes -- for the class of errors memory can fix. Paired A/B benchmark
103
- (`benchmarks/bench_error_reduction.py`): the same model (claude-haiku-4-5)
104
- runs 96 tasks twice, with and without errlore injection. Deterministic
105
- validators, no LLM judges; raw outputs committed in
106
- [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/).
118
+ For the class of errors memory can fix yes, and here's the honest version.
119
+ Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
120
+ (claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
121
+ Deterministic validators, no LLM judges; raw outputs committed in
122
+ [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
123
+ you can recompute every number.
107
124
 
108
125
  | arm | failures | fail rate |
109
126
  |-----|----------|-----------|
@@ -114,14 +131,24 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
114
131
  Split by error class:
115
132
 
116
133
  - **Knowledge-gap errors** (workspace conventions: date formats, ID
117
- normalization, rounding rules, CSV column order): **46/48 -> 0/48, a 100%
118
- reduction.** The model didn't know the convention; a lesson told it.
134
+ normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
135
+ can't know a convention it was never told, so arm A fails almost by
136
+ construction; the result shows errlore **captures the fix once and re-supplies
137
+ it** on the next similar task, end to end. That store-and-inject loop is the
138
+ claim — not that memory teaches skills.
119
139
  - **Capability-gap errors** (letter counting, string reversal): 17/48 ->
120
140
  20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
121
141
  model doesn't know, not what it can't do.
122
142
 
123
- Reproduce: `python benchmarks/bench_error_reduction.py` (needs an Anthropic
124
- API key; the task families and validators ship in the repo).
143
+ **Caveats, up front:** this is a single run at temperature 0 (LLM output is
144
+ still slightly non-deterministic, so exact fine-grained counts vary between
145
+ runs — the large knowledge-gap effect is robust; the capability-gap delta is
146
+ within noise). The knowledge-gap task families use conventions the model
147
+ demonstrably can't guess, which is the point — but it means the headline is
148
+ "the loop works," not "90% fewer errors everywhere."
149
+
150
+ Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
151
+ (needs an Anthropic API key; task families and validators ship in the repo).
125
152
 
126
153
  ## How it works
127
154
 
@@ -146,13 +173,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
146
173
  type before, `inject_for` adds a warning block to the prompt. Separate from
147
174
  lessons: lessons are *solutions*, known issues are *warnings*.
148
175
 
149
- ### 3. Trust loop
176
+ ### 3. Trust loop *(experimental)*
150
177
 
151
178
  Bayesian per-model weights with adaptive learning rate, cold-start blending,
152
179
  entropy enforcement, and temporal decay. After enough observations, call
153
180
  `mem.best_model("code_generation")` to pick the model that historically
154
181
  performs best on that domain.
155
182
 
183
+ > **Status: experimental.** The engine is tested and works, but discrimination
184
+ > between models only emerges from a *spread* of real outcomes over time — feed
185
+ > it a stream that is mostly successes and every model converges near the cap.
186
+ > Treat `best_model()` as a hint to validate, not a proven router yet. The
187
+ > lesson + known-issue loops above are the proven core (see the A/B benchmark).
188
+
156
189
  ## Semantic retrieval (optional)
157
190
 
158
191
  By default, errlore finds relevant lessons via word overlap (zero
@@ -167,6 +200,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
167
200
  mem = AgentMemory("./agent_memory", embeddings=True)
168
201
  ```
169
202
 
203
+ > The embedding model (~120 MB ONNX) is downloaded once on first use, then
204
+ > runs locally with no further network calls. The core (word-overlap) stays
205
+ > fully offline and dependency-free.
206
+
170
207
  ### Benchmark (adversarial paraphrasing)
171
208
 
172
209
  Tested on 40 lessons with adversarially paraphrased queries
@@ -188,15 +225,37 @@ queries with shared vocabulary, word-overlap works fine.
188
225
  errlore is framework-agnostic. It produces a text block; you put it in the
189
226
  system prompt.
190
227
 
191
- | Provider | Example |
192
- |------------|------------------------------------------------------|
193
- | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
194
- | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
195
- | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
228
+ **Claude Code** — one command wires up failure-memory across sessions:
229
+
230
+ ```bash
231
+ errlore init claude-code # or: --project for this repo only
232
+ ```
233
+
234
+ Failed Bash commands become lessons; every new session is briefed on past
235
+ pitfalls. See [examples/claude-code/](examples/claude-code/).
236
+
237
+ | Provider | Example |
238
+ |-------------|------------------------------------------------------|
239
+ | Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
240
+ | Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
241
+ | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
242
+ | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
243
+ | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
196
244
 
197
- All examples run offline with `python examples/<name>.py` (mock responses,
245
+ The SDK examples run offline with `python examples/<name>.py` (mock responses,
198
246
  no API keys). Set `use_api=True` to call real models.
199
247
 
248
+ ### CLI
249
+
250
+ `pip install errlore` also installs an `errlore` command:
251
+
252
+ ```bash
253
+ errlore init claude-code # install Claude Code hooks + settings
254
+ errlore stats # memory stats for a data dir (--data-dir)
255
+ errlore lessons # list stored lessons
256
+ errlore --version
257
+ ```
258
+
200
259
  ## API overview
201
260
 
202
261
  The main entry point is `AgentMemory`. All other classes are internal --
@@ -210,7 +269,7 @@ you only need them for advanced use.
210
269
  | `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
211
270
  | `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
212
271
  | `lessons(limit)` | List all lessons (sorted by confidence). |
213
- | `best_model(domain)` | Model with the highest trust weight. |
272
+ | `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
214
273
  | `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
215
274
  | `pending_injections()` | Injections not yet reported. |
216
275
  | `stats()` | Aggregate counts + trust weights. |
@@ -236,6 +295,49 @@ you only need them for advanced use.
236
295
  (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
237
296
  (embedding metadata), `trust.json` (trust engine state).
238
297
 
298
+ ## Security
299
+
300
+ A lesson is **trusted prompt content by design** — it is injected into your
301
+ prompts and reaches the model. So:
302
+
303
+ - **Do not ingest lessons from untrusted sources without review.** Treat lesson
304
+ capture like a code review, not like user input. A malicious lesson is a
305
+ prompt-injection vector — and this is the real control, not the sanitizer.
306
+ - **What the sanitizer does (and does not) do.** The lesson *pattern* passes
307
+ `sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
308
+ so log blobs don't pollute the prompt. It is a noise filter, **not** an
309
+ injection defense — it does not neutralize natural-language instructions, and
310
+ the *solution* text is stored as you author it (so it can hold real code).
311
+ Don't rely on it to make untrusted lessons safe.
312
+ - You control what becomes a lesson (`resolve(..., lesson=...)` /
313
+ `add_lesson(...)`); nothing is auto-promoted from raw model output.
314
+
315
+ Report security issues to the address in [SECURITY.md](SECURITY.md).
316
+
317
+ ## Scale & limits (honest)
318
+
319
+ errlore is built for **one process, thousands of lessons** — a single agent or
320
+ a coding-agent session, not a high-throughput fleet. Know the edges:
321
+
322
+ - **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
323
+ ledger each call, so at very high injection volumes it slows down (roughly
324
+ linear in total injections). Fine for interactive/agent use; log compaction
325
+ is the next roadmap item. If you don't need the reinforcement loop, you can
326
+ ignore `report_outcome` and the file stays small.
327
+ - **Single-process by default.** The lesson/error stores use cross-process file
328
+ locks and are safe to share, but the **trust engine and the optional vector
329
+ index are not cross-process safe** — two processes writing `trust.json` /
330
+ `vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
331
+ writer, or give each process its own `data_dir`. Multi-agent shared memory is
332
+ on the roadmap.
333
+ - **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
334
+ over a large existing lesson store is slow the first time (then incremental).
335
+ - Concurrency is tested across threads; **multi-process** stress is not yet in
336
+ the suite.
337
+
338
+ None of these bite at the scale errlore targets today; they're stated so you
339
+ can decide, not discover.
340
+
239
341
  ## Roadmap
240
342
 
241
343
  - [ ] Log compaction for injections journal
@@ -18,14 +18,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
18
18
  into the prompt for similar future tasks.
19
19
  - **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
20
20
  extraction tasks") injected as warnings.
21
- - **Trust** -- Bayesian per-model, per-domain trust weights: know which model to pick
22
- for which job, based on observed outcomes.
21
+ - **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
22
+ point for which model to pick per job, based on observed outcomes. Needs a spread of
23
+ real outcomes to separate models; shipped, but not yet proven on production traffic.
23
24
  - **Closed loop** -- errlore tracks whether an injected lesson actually helped and
24
25
  reinforces or decays it automatically.
25
26
 
26
27
  Embedded, file-based (JSONL), no server, no database, no API keys required.
27
28
  Works fully offline. Your data never leaves your machine.
28
29
 
30
+ ## Who it's for
31
+
32
+ errlore isn't memory for everything — it's memory for **failures**. It shines
33
+ wherever an agent repeats the *same class* of mistake:
34
+
35
+ - **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
36
+ the same bug or forgetting a project convention across sessions.
37
+ - **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
38
+ same date format, rounding rule, or schema field every week.
39
+ - **Any repeated-failure workflow** where a fix should stick the first time,
40
+ not be re-discovered on every run.
41
+
42
+ It fixes what the model doesn't *know* (a convention, a gotcha), not what it
43
+ *can't do* — see the benchmark below.
44
+
29
45
  ## Quickstart (< 5 minutes)
30
46
 
31
47
  ```bash
@@ -65,11 +81,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
65
81
 
66
82
  ## Does it actually reduce errors?
67
83
 
68
- Yes -- for the class of errors memory can fix. Paired A/B benchmark
69
- (`benchmarks/bench_error_reduction.py`): the same model (claude-haiku-4-5)
70
- runs 96 tasks twice, with and without errlore injection. Deterministic
71
- validators, no LLM judges; raw outputs committed in
72
- [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/).
84
+ For the class of errors memory can fix yes, and here's the honest version.
85
+ Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
86
+ (claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
87
+ Deterministic validators, no LLM judges; raw outputs committed in
88
+ [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
89
+ you can recompute every number.
73
90
 
74
91
  | arm | failures | fail rate |
75
92
  |-----|----------|-----------|
@@ -80,14 +97,24 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
80
97
  Split by error class:
81
98
 
82
99
  - **Knowledge-gap errors** (workspace conventions: date formats, ID
83
- normalization, rounding rules, CSV column order): **46/48 -> 0/48, a 100%
84
- reduction.** The model didn't know the convention; a lesson told it.
100
+ normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
101
+ can't know a convention it was never told, so arm A fails almost by
102
+ construction; the result shows errlore **captures the fix once and re-supplies
103
+ it** on the next similar task, end to end. That store-and-inject loop is the
104
+ claim — not that memory teaches skills.
85
105
  - **Capability-gap errors** (letter counting, string reversal): 17/48 ->
86
106
  20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
87
107
  model doesn't know, not what it can't do.
88
108
 
89
- Reproduce: `python benchmarks/bench_error_reduction.py` (needs an Anthropic
90
- API key; the task families and validators ship in the repo).
109
+ **Caveats, up front:** this is a single run at temperature 0 (LLM output is
110
+ still slightly non-deterministic, so exact fine-grained counts vary between
111
+ runs — the large knowledge-gap effect is robust; the capability-gap delta is
112
+ within noise). The knowledge-gap task families use conventions the model
113
+ demonstrably can't guess, which is the point — but it means the headline is
114
+ "the loop works," not "90% fewer errors everywhere."
115
+
116
+ Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
117
+ (needs an Anthropic API key; task families and validators ship in the repo).
91
118
 
92
119
  ## How it works
93
120
 
@@ -112,13 +139,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
112
139
  type before, `inject_for` adds a warning block to the prompt. Separate from
113
140
  lessons: lessons are *solutions*, known issues are *warnings*.
114
141
 
115
- ### 3. Trust loop
142
+ ### 3. Trust loop *(experimental)*
116
143
 
117
144
  Bayesian per-model weights with adaptive learning rate, cold-start blending,
118
145
  entropy enforcement, and temporal decay. After enough observations, call
119
146
  `mem.best_model("code_generation")` to pick the model that historically
120
147
  performs best on that domain.
121
148
 
149
+ > **Status: experimental.** The engine is tested and works, but discrimination
150
+ > between models only emerges from a *spread* of real outcomes over time — feed
151
+ > it a stream that is mostly successes and every model converges near the cap.
152
+ > Treat `best_model()` as a hint to validate, not a proven router yet. The
153
+ > lesson + known-issue loops above are the proven core (see the A/B benchmark).
154
+
122
155
  ## Semantic retrieval (optional)
123
156
 
124
157
  By default, errlore finds relevant lessons via word overlap (zero
@@ -133,6 +166,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
133
166
  mem = AgentMemory("./agent_memory", embeddings=True)
134
167
  ```
135
168
 
169
+ > The embedding model (~120 MB ONNX) is downloaded once on first use, then
170
+ > runs locally with no further network calls. The core (word-overlap) stays
171
+ > fully offline and dependency-free.
172
+
136
173
  ### Benchmark (adversarial paraphrasing)
137
174
 
138
175
  Tested on 40 lessons with adversarially paraphrased queries
@@ -154,15 +191,37 @@ queries with shared vocabulary, word-overlap works fine.
154
191
  errlore is framework-agnostic. It produces a text block; you put it in the
155
192
  system prompt.
156
193
 
157
- | Provider | Example |
158
- |------------|------------------------------------------------------|
159
- | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
160
- | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
161
- | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
194
+ **Claude Code** — one command wires up failure-memory across sessions:
195
+
196
+ ```bash
197
+ errlore init claude-code # or: --project for this repo only
198
+ ```
199
+
200
+ Failed Bash commands become lessons; every new session is briefed on past
201
+ pitfalls. See [examples/claude-code/](examples/claude-code/).
202
+
203
+ | Provider | Example |
204
+ |-------------|------------------------------------------------------|
205
+ | Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
206
+ | Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
207
+ | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
208
+ | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
209
+ | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
162
210
 
163
- All examples run offline with `python examples/<name>.py` (mock responses,
211
+ The SDK examples run offline with `python examples/<name>.py` (mock responses,
164
212
  no API keys). Set `use_api=True` to call real models.
165
213
 
214
+ ### CLI
215
+
216
+ `pip install errlore` also installs an `errlore` command:
217
+
218
+ ```bash
219
+ errlore init claude-code # install Claude Code hooks + settings
220
+ errlore stats # memory stats for a data dir (--data-dir)
221
+ errlore lessons # list stored lessons
222
+ errlore --version
223
+ ```
224
+
166
225
  ## API overview
167
226
 
168
227
  The main entry point is `AgentMemory`. All other classes are internal --
@@ -176,7 +235,7 @@ you only need them for advanced use.
176
235
  | `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
177
236
  | `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
178
237
  | `lessons(limit)` | List all lessons (sorted by confidence). |
179
- | `best_model(domain)` | Model with the highest trust weight. |
238
+ | `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
180
239
  | `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
181
240
  | `pending_injections()` | Injections not yet reported. |
182
241
  | `stats()` | Aggregate counts + trust weights. |
@@ -202,6 +261,49 @@ you only need them for advanced use.
202
261
  (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
203
262
  (embedding metadata), `trust.json` (trust engine state).
204
263
 
264
+ ## Security
265
+
266
+ A lesson is **trusted prompt content by design** — it is injected into your
267
+ prompts and reaches the model. So:
268
+
269
+ - **Do not ingest lessons from untrusted sources without review.** Treat lesson
270
+ capture like a code review, not like user input. A malicious lesson is a
271
+ prompt-injection vector — and this is the real control, not the sanitizer.
272
+ - **What the sanitizer does (and does not) do.** The lesson *pattern* passes
273
+ `sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
274
+ so log blobs don't pollute the prompt. It is a noise filter, **not** an
275
+ injection defense — it does not neutralize natural-language instructions, and
276
+ the *solution* text is stored as you author it (so it can hold real code).
277
+ Don't rely on it to make untrusted lessons safe.
278
+ - You control what becomes a lesson (`resolve(..., lesson=...)` /
279
+ `add_lesson(...)`); nothing is auto-promoted from raw model output.
280
+
281
+ Report security issues to the address in [SECURITY.md](SECURITY.md).
282
+
283
+ ## Scale & limits (honest)
284
+
285
+ errlore is built for **one process, thousands of lessons** — a single agent or
286
+ a coding-agent session, not a high-throughput fleet. Know the edges:
287
+
288
+ - **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
289
+ ledger each call, so at very high injection volumes it slows down (roughly
290
+ linear in total injections). Fine for interactive/agent use; log compaction
291
+ is the next roadmap item. If you don't need the reinforcement loop, you can
292
+ ignore `report_outcome` and the file stays small.
293
+ - **Single-process by default.** The lesson/error stores use cross-process file
294
+ locks and are safe to share, but the **trust engine and the optional vector
295
+ index are not cross-process safe** — two processes writing `trust.json` /
296
+ `vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
297
+ writer, or give each process its own `data_dir`. Multi-agent shared memory is
298
+ on the roadmap.
299
+ - **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
300
+ over a large existing lesson store is slow the first time (then incremental).
301
+ - Concurrency is tested across threads; **multi-process** stress is not yet in
302
+ the suite.
303
+
304
+ None of these bite at the scale errlore targets today; they're stated so you
305
+ can decide, not discover.
306
+
205
307
  ## Roadmap
206
308
 
207
309
  - [ ] Log compaction for injections journal
@@ -16,15 +16,15 @@ runs end-to-end without network access.
16
16
 
17
17
  from __future__ import annotations
18
18
 
19
+ import os
19
20
  import tempfile
20
21
  from pathlib import Path
21
22
 
22
23
  from errlore import AgentMemory
23
24
 
24
- # Current Claude lineup (2026): claude-fable-5 (most capable),
25
- # claude-opus-4-8 (recommended default), claude-sonnet-4-6 (speed/cost balance),
26
- # claude-haiku-4-5 (fastest). Use exact IDs as-is -- no date suffixes.
27
- MODEL = "claude-opus-4-8"
25
+ # Model is just a label to errlore (it never calls the API itself). Override
26
+ # with your own Claude model id; the default is a small, fast, cheap option.
27
+ MODEL = os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-latest")
28
28
 
29
29
 
30
30
  # -- Agent wrapper -----------------------------------------------------------
@@ -0,0 +1,37 @@
1
+ # errlore + Claude Code
2
+
3
+ Give your coding agent a memory of its own failures across sessions:
4
+
5
+ - **PostToolUse hook** — every failed Bash command is logged into errlore.
6
+ Resolve the ones you fixed (`mem.resolve(err_id, ..., lesson=...)`) or use
7
+ `mem.add_lesson()` to capture takeaways directly.
8
+ - **SessionStart hook** — each new session begins with a briefing block of
9
+ relevant lessons and per-tool KNOWN ISSUES, printed into the context.
10
+
11
+ ## Setup (one command)
12
+
13
+ ```bash
14
+ pip install errlore
15
+ errlore init claude-code # global (~/.claude/settings.json)
16
+ errlore init claude-code --project # or this repo only (./.claude/settings.json)
17
+ ```
18
+
19
+ That writes the two hook scripts (to `~/.errlore/hooks/`) and merges them into
20
+ your `settings.json` — idempotently, preserving any hooks you already have.
21
+ Restart Claude Code (or open a new session) to pick them up. Options:
22
+ `--data-dir` (where the memory lives, default `~/.errlore/claude-code`) and
23
+ `--hooks-dir`.
24
+
25
+ Handy afterwards: `errlore stats` and `errlore lessons` to see what it learned.
26
+
27
+ ### Manual setup (if you'd rather wire it yourself)
28
+
29
+ The scripts in this folder (`errlore_posttooluse.py`, `errlore_sessionstart.py`)
30
+ plus `settings.json.example` show the shape: copy them somewhere stable, fix the
31
+ paths, and merge into `.claude/settings.json`. `export ERRLORE_DATA=...` picks
32
+ the memory dir.
33
+
34
+ Notes: hook event field names can differ between Claude Code versions —
35
+ the PostToolUse script reads them defensively and never breaks the agent
36
+ loop (exit 0 on anything unexpected). Check `claude --help` / the hooks
37
+ docs for your version if events don't arrive.
@@ -19,14 +19,15 @@ The ``if __name__`` block uses a mock instead of a real LLM call.
19
19
 
20
20
  from __future__ import annotations
21
21
 
22
+ import os
22
23
  import tempfile
23
24
  from pathlib import Path
24
25
 
25
26
  from errlore import AgentMemory, Injection
26
27
 
27
- # Current OpenAI lineup (2026): gpt-5.5 (frontier, recommended),
28
- # gpt-5.4-mini / gpt-5.4-nano (lower latency & cost).
29
- MODEL = "gpt-5.5"
28
+ # Model is just a label to errlore (it never calls the API itself). Override
29
+ # with your own; the default is a small, cheap, widely-available option.
30
+ MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
30
31
 
31
32
 
32
33
  # -- errlore + LangChain glue ------------------------------------------------
@@ -16,14 +16,15 @@ runs end-to-end without network access.
16
16
 
17
17
  from __future__ import annotations
18
18
 
19
+ import os
19
20
  import tempfile
20
21
  from pathlib import Path
21
22
 
22
23
  from errlore import AgentMemory
23
24
 
24
- # Current OpenAI lineup (2026): gpt-5.5 (frontier, recommended),
25
- # gpt-5.4-mini / gpt-5.4-nano (lower latency & cost).
26
- MODEL = "gpt-5.5"
25
+ # Model is just a label to errlore (it never calls the API itself). Override
26
+ # with your own; the default is a small, cheap, widely-available option.
27
+ MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
27
28
 
28
29
 
29
30
  # -- Agent wrapper -----------------------------------------------------------
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "errlore"
7
- version = "0.1.3"
7
+ version = "0.1.4"
8
8
  description = "Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -24,6 +24,9 @@ dependencies = [
24
24
  "filelock>=3.13",
25
25
  ]
26
26
 
27
+ [project.scripts]
28
+ errlore = "errlore.cli:main"
29
+
27
30
  [project.optional-dependencies]
28
31
  embeddings = [
29
32
  "fastembed>=0.6,<0.9",