errlore 0.1.3__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. errlore-0.2.0/.github/workflows/release.yml +74 -0
  2. {errlore-0.1.3 → errlore-0.2.0}/CHANGELOG.md +54 -0
  3. {errlore-0.1.3 → errlore-0.2.0}/PKG-INFO +135 -21
  4. {errlore-0.1.3 → errlore-0.2.0}/README.md +133 -19
  5. {errlore-0.1.3 → errlore-0.2.0}/benchmarks/bench_error_reduction.py +60 -2
  6. errlore-0.2.0/benchmarks/results/REPRODUCIBILITY_2026-07-11.md +105 -0
  7. errlore-0.2.0/benchmarks/results/gemma_2026-07-11_report.txt +37 -0
  8. errlore-0.2.0/benchmarks/results/haiku_2026-07-11_report.txt +35 -0
  9. errlore-0.2.0/benchmarks/results/haiku_2026-07-11_seed11.txt +35 -0
  10. errlore-0.2.0/benchmarks/results/haiku_2026-07-11_seed22.txt +35 -0
  11. errlore-0.2.0/benchmarks/results/haiku_2026-07-11_seed33.txt +35 -0
  12. errlore-0.2.0/benchmarks/results/haiku_newfamilies_2026-07-11_report.txt +27 -0
  13. {errlore-0.1.3 → errlore-0.2.0}/examples/anthropic_agent.py +4 -4
  14. errlore-0.2.0/examples/claude-code/README.md +37 -0
  15. {errlore-0.1.3 → errlore-0.2.0}/examples/langchain_agent.py +4 -3
  16. {errlore-0.1.3 → errlore-0.2.0}/examples/openai_agent.py +4 -3
  17. {errlore-0.1.3 → errlore-0.2.0}/pyproject.toml +5 -2
  18. {errlore-0.1.3 → errlore-0.2.0}/site/index.html +99 -26
  19. {errlore-0.1.3 → errlore-0.2.0}/site/llms.txt +9 -6
  20. errlore-0.2.0/site/robots.txt +12 -0
  21. errlore-0.2.0/site/sitemap.xml +9 -0
  22. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/__init__.py +1 -1
  23. errlore-0.2.0/src/errlore/cli.py +200 -0
  24. errlore-0.2.0/src/errlore/integrations/__init__.py +5 -0
  25. errlore-0.2.0/src/errlore/integrations/claude_code.py +92 -0
  26. errlore-0.2.0/tests/test_cli.py +145 -0
  27. errlore-0.1.3/examples/claude-code/README.md +0 -23
  28. {errlore-0.1.3 → errlore-0.2.0}/.github/workflows/ci.yml +0 -0
  29. {errlore-0.1.3 → errlore-0.2.0}/.gitignore +0 -0
  30. {errlore-0.1.3 → errlore-0.2.0}/LICENSE +0 -0
  31. {errlore-0.1.3 → errlore-0.2.0}/SECURITY.md +0 -0
  32. {errlore-0.1.3 → errlore-0.2.0}/benchmarks/bench_retrieval.py +0 -0
  33. {errlore-0.1.3 → errlore-0.2.0}/benchmarks/results/error_reduction/report.md +0 -0
  34. {errlore-0.1.3 → errlore-0.2.0}/examples/claude-code/errlore_posttooluse.py +0 -0
  35. {errlore-0.1.3 → errlore-0.2.0}/examples/claude-code/errlore_sessionstart.py +0 -0
  36. {errlore-0.1.3 → errlore-0.2.0}/examples/claude-code/settings.json.example +0 -0
  37. {errlore-0.1.3 → errlore-0.2.0}/integrations/openwebui/README.md +0 -0
  38. {errlore-0.1.3 → errlore-0.2.0}/integrations/openwebui/errlore_feedback_action.py +0 -0
  39. {errlore-0.1.3 → errlore-0.2.0}/integrations/openwebui/errlore_memory_filter.py +0 -0
  40. {errlore-0.1.3 → errlore-0.2.0}/site/demo.gif +0 -0
  41. {errlore-0.1.3 → errlore-0.2.0}/site/demo_script.py +0 -0
  42. {errlore-0.1.3 → errlore-0.2.0}/site/og.png +0 -0
  43. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/__init__.py +0 -0
  44. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/classifier.py +0 -0
  45. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/injector.py +0 -0
  46. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/patterns.py +0 -0
  47. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/tracker.py +0 -0
  48. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/facade.py +0 -0
  49. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/__init__.py +0 -0
  50. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/jsonl_index.py +0 -0
  51. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/jsonl_writer.py +0 -0
  52. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/repair.py +0 -0
  53. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/lessons/__init__.py +0 -0
  54. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/lessons/models.py +0 -0
  55. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/lessons/store.py +0 -0
  56. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/py.typed +0 -0
  57. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/retrieval/__init__.py +0 -0
  58. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/retrieval/backend.py +0 -0
  59. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/retrieval/index.py +0 -0
  60. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/sanitize.py +0 -0
  61. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/trust/__init__.py +0 -0
  62. {errlore-0.1.3 → errlore-0.2.0}/src/errlore/trust/engine.py +0 -0
  63. {errlore-0.1.3 → errlore-0.2.0}/tests/conftest.py +0 -0
  64. {errlore-0.1.3 → errlore-0.2.0}/tests/test_errmem.py +0 -0
  65. {errlore-0.1.3 → errlore-0.2.0}/tests/test_facade.py +0 -0
  66. {errlore-0.1.3 → errlore-0.2.0}/tests/test_io.py +0 -0
  67. {errlore-0.1.3 → errlore-0.2.0}/tests/test_lessons.py +0 -0
  68. {errlore-0.1.3 → errlore-0.2.0}/tests/test_openwebui_integration.py +0 -0
  69. {errlore-0.1.3 → errlore-0.2.0}/tests/test_regressions.py +0 -0
  70. {errlore-0.1.3 → errlore-0.2.0}/tests/test_retrieval.py +0 -0
  71. {errlore-0.1.3 → errlore-0.2.0}/tests/test_trust.py +0 -0
@@ -0,0 +1,74 @@
1
+ name: release
2
+
3
+ # Tag-driven release: push a v* tag -> full test gate -> build sdist+wheel ->
4
+ # publish to PyPI via trusted publishing (OIDC; no long-lived token in CI).
5
+ #
6
+ # One-time PyPI setup (owner): pypi.org -> project errlore -> Publishing ->
7
+ # add GitHub publisher (owner=Ma4etaSS, repo=errlore, workflow=release.yml,
8
+ # environment=pypi). Until that's configured the publish step fails cleanly
9
+ # and the build artifacts are still attached to the GitHub release.
10
+
11
+ on:
12
+ push:
13
+ tags: ["v*"]
14
+
15
+ jobs:
16
+ test:
17
+ runs-on: ubuntu-latest
18
+ steps:
19
+ - uses: actions/checkout@v4
20
+ - uses: actions/setup-python@v5
21
+ with:
22
+ python-version: "3.12"
23
+ - run: pip install -e .[dev]
24
+ - run: ruff check .
25
+ - run: mypy
26
+ - run: pytest --cov=errlore --cov-fail-under=80
27
+
28
+ build:
29
+ needs: test
30
+ runs-on: ubuntu-latest
31
+ steps:
32
+ - uses: actions/checkout@v4
33
+ - uses: actions/setup-python@v5
34
+ with:
35
+ python-version: "3.12"
36
+ - run: pip install build
37
+ - run: python -m build
38
+ - uses: actions/upload-artifact@v4
39
+ with:
40
+ name: dist
41
+ path: dist/
42
+
43
+ publish:
44
+ needs: build
45
+ runs-on: ubuntu-latest
46
+ environment: pypi
47
+ permissions:
48
+ id-token: write # OIDC for PyPI trusted publishing
49
+ steps:
50
+ - uses: actions/download-artifact@v4
51
+ with:
52
+ name: dist
53
+ path: dist/
54
+ - uses: pypa/gh-action-pypi-publish@release/v1
55
+
56
+ github-release:
57
+ needs: build
58
+ runs-on: ubuntu-latest
59
+ permissions:
60
+ contents: write
61
+ steps:
62
+ - uses: actions/checkout@v4
63
+ - uses: actions/download-artifact@v4
64
+ with:
65
+ name: dist
66
+ path: dist/
67
+ - name: Create GitHub release with artifacts
68
+ env:
69
+ GH_TOKEN: ${{ github.token }}
70
+ run: |
71
+ gh release create "${GITHUB_REF_NAME}" dist/* \
72
+ --title "${GITHUB_REF_NAME}" \
73
+ --notes "See CHANGELOG.md for details." || \
74
+ gh release upload "${GITHUB_REF_NAME}" dist/* --clobber
@@ -5,6 +5,60 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [0.2.0] - 2026-07-11 — "The Proof Release"
9
+
10
+ The headline benchmark is now proven on all three generality axes:
11
+ **seed-robustness** (5 independent runs), **task-generality** (fresh realistic
12
+ convention families), and **model-diversity** (Anthropic + Gemma). Development
13
+ status bumped Alpha → Beta.
14
+
15
+ ### Added
16
+ - **Reproducibility evidence for the error-reduction benchmark**
17
+ (`benchmarks/results/REPRODUCIBILITY_2026-07-11.md`): the headline A/B is now
18
+ reproduced across 5 independent runs on claude-haiku-4-5 (two default-seed
19
+ runs 5 days apart + three fresh RNG seeds). Every run: 66.7–69.8% repeat-error
20
+ reduction, exact McNemar p between 8.4e-12 and 1.8e-9, knowledge-gap reduction
21
+ 95–100%, capability-gap −12% to 0%. Resolves the prior "single run" caveat in
22
+ the README.
23
+ - `bench_error_reduction.py` reads `BENCH_RNG_SEED` from the environment so the
24
+ same task families can be re-drawn with different instances for seed-robustness
25
+ checks. Also `BENCH_FAMILIES` / `BENCH_SEED_N` / `BENCH_TEST_N` for narrowing a
26
+ run (e.g. a small cross-family probe on a tight free-tier quota).
27
+ - **Task-generality: two new realistic knowledge-gap families** (`status_code` —
28
+ an arbitrary internal status enum; `branch_name` — a non-standard git branch
29
+ convention). On claude-haiku-4-5 both go 100% → 0% (24/24 → 0/24, McNemar
30
+ p=1.19e-07), confirming the store-and-inject effect isn't specific to the
31
+ original toy families. Report + raw outputs in `benchmarks/results/`.
32
+ - **Model-diversity: full grid on gemma-4-31b** (Cerebras — a different model
33
+ family): 66.7% → 20.0% fail (70.0% reduction, McNemar p=2.6e-13),
34
+ knowledge-gap 83%, capability-gap −20%; branch_name and status_code flip
35
+ 100%→0% here too. One honest wrinkle documented: `csv_order` does not
36
+ transfer to gemma (12→12) while it flips to 0 on Haiku — lesson-following is
37
+ itself model-dependent at the margin. `BENCH_MODEL` env override added for
38
+ running any backend against a specific model.
39
+
40
+ ## [0.1.4] - 2026-07-09
41
+
42
+ ### Added
43
+ - **CLI** (`errlore` console command): `errlore init claude-code` writes the
44
+ two Claude Code hook scripts and idempotently merges them into your
45
+ `settings.json` (global or `--project`), preserving existing hooks — a
46
+ one-command install instead of copy/edit/merge. Plus `errlore stats` and
47
+ `errlore lessons`.
48
+ - `errlore.integrations.claude_code` — the hook logic (`post_tool_use`,
49
+ `session_start`) now ships in the package and is tested, so the generated
50
+ hooks are 3-line shims.
51
+
52
+ ### Changed
53
+ - README: honest A/B framing (the knowledge-gap baseline fails by
54
+ construction; the result shows the capture-and-re-supply loop works, not
55
+ that memory teaches skills; single-run-at-temp-0 caveat). Coding-agent-first
56
+ hero. Security section reworded — the sanitizer is a noise filter on the
57
+ pattern, not an injection defense. Added a "Scale & limits" section
58
+ (unbounded injections journal, single-process trust/vector index).
59
+ - Examples no longer hard-code aging frontier model ids; they use
60
+ `os.getenv(...)` with a small default.
61
+
8
62
  ## [0.1.3] - 2026-07-06
9
63
 
10
64
  ### Fixed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: errlore
3
- Version: 0.1.3
3
+ Version: 0.2.0
4
4
  Summary: Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server.
5
5
  Project-URL: Homepage, https://errlore.com
6
6
  Project-URL: Repository, https://github.com/Ma4etaSS/errlore
@@ -10,7 +10,7 @@ Author: Ma4etaSS
10
10
  License-Expression: MIT
11
11
  License-File: LICENSE
12
12
  Keywords: agents,error-learning,lessons,llm,memory,trust
13
- Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Development Status :: 4 - Beta
14
14
  Classifier: Intended Audience :: Developers
15
15
  Classifier: Programming Language :: Python :: 3.10
16
16
  Classifier: Programming Language :: Python :: 3.11
@@ -52,14 +52,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
52
52
  into the prompt for similar future tasks.
53
53
  - **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
54
54
  extraction tasks") injected as warnings.
55
- - **Trust** -- Bayesian per-model, per-domain trust weights: know which model to pick
56
- for which job, based on observed outcomes.
55
+ - **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
56
+ point for which model to pick per job, based on observed outcomes. Needs a spread of
57
+ real outcomes to separate models; shipped, but not yet proven on production traffic.
57
58
  - **Closed loop** -- errlore tracks whether an injected lesson actually helped and
58
59
  reinforces or decays it automatically.
59
60
 
60
61
  Embedded, file-based (JSONL), no server, no database, no API keys required.
61
62
  Works fully offline. Your data never leaves your machine.
62
63
 
64
+ ## Who it's for
65
+
66
+ errlore isn't memory for everything — it's memory for **failures**. It shines
67
+ wherever an agent repeats the *same class* of mistake:
68
+
69
+ - **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
70
+ the same bug or forgetting a project convention across sessions.
71
+ - **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
72
+ same date format, rounding rule, or schema field every week.
73
+ - **Any repeated-failure workflow** where a fix should stick the first time,
74
+ not be re-discovered on every run.
75
+
76
+ It fixes what the model doesn't *know* (a convention, a gotcha), not what it
77
+ *can't do* — see the benchmark below.
78
+
63
79
  ## Quickstart (< 5 minutes)
64
80
 
65
81
  ```bash
@@ -99,11 +115,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
99
115
 
100
116
  ## Does it actually reduce errors?
101
117
 
102
- Yes -- for the class of errors memory can fix. Paired A/B benchmark
103
- (`benchmarks/bench_error_reduction.py`): the same model (claude-haiku-4-5)
104
- runs 96 tasks twice, with and without errlore injection. Deterministic
105
- validators, no LLM judges; raw outputs committed in
106
- [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/).
118
+ For the class of errors memory can fix yes, and here's the honest version.
119
+ Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
120
+ (claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
121
+ Deterministic validators, no LLM judges; raw outputs committed in
122
+ [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
123
+ you can recompute every number.
107
124
 
108
125
  | arm | failures | fail rate |
109
126
  |-----|----------|-----------|
@@ -114,14 +131,36 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
114
131
  Split by error class:
115
132
 
116
133
  - **Knowledge-gap errors** (workspace conventions: date formats, ID
117
- normalization, rounding rules, CSV column order): **46/48 -> 0/48, a 100%
118
- reduction.** The model didn't know the convention; a lesson told it.
134
+ normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
135
+ can't know a convention it was never told, so arm A fails almost by
136
+ construction; the result shows errlore **captures the fix once and re-supplies
137
+ it** on the next similar task, end to end. That store-and-inject loop is the
138
+ claim — not that memory teaches skills.
119
139
  - **Capability-gap errors** (letter counting, string reversal): 17/48 ->
120
140
  20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
121
141
  model doesn't know, not what it can't do.
122
142
 
123
- Reproduce: `python benchmarks/bench_error_reduction.py` (needs an Anthropic
124
- API key; the task families and validators ship in the repo).
143
+ **Reproduced across 5 independent runs** (two on the default seed 5 days apart,
144
+ plus three fresh RNG seeds different task instances). Every run: overall
145
+ reduction **66.7–69.8%**, exact McNemar **p between 8.4e-12 and 1.8e-9**,
146
+ knowledge-gap reduction **95–100%**, capability-gap **−12% to 0%** (no help).
147
+ **Cross-model:** the same grid on **gemma-4-31b** (a different model family)
148
+ lands at **70.0% reduction** (66.7% → 20.0%, p = 2.6e-13), knowledge-gap 83%,
149
+ capability-gap −20% — same effect, same honest boundary. **Task-generality:**
150
+ two fresh realistic-convention families (an arbitrary internal status enum and
151
+ a non-standard git branch convention) go **100% → 0%** on both models.
152
+ Full table + per-run reports:
153
+ [benchmarks/results/REPRODUCIBILITY_2026-07-11.md](benchmarks/results/REPRODUCIBILITY_2026-07-11.md).
154
+
155
+ **Caveats, up front:** temperature 0 still leaves LLM output slightly
156
+ non-deterministic, so exact fine-grained counts vary run to run — the large
157
+ knowledge-gap effect is robust across all five runs; the capability-gap delta
158
+ stays within noise. The knowledge-gap task families use conventions the model
159
+ demonstrably can't guess, which is the point — but it means the headline is
160
+ "the loop works," not "90% fewer errors everywhere."
161
+
162
+ Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
163
+ (needs an Anthropic API key; task families and validators ship in the repo).
125
164
 
126
165
  ## How it works
127
166
 
@@ -146,13 +185,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
146
185
  type before, `inject_for` adds a warning block to the prompt. Separate from
147
186
  lessons: lessons are *solutions*, known issues are *warnings*.
148
187
 
149
- ### 3. Trust loop
188
+ ### 3. Trust loop *(experimental)*
150
189
 
151
190
  Bayesian per-model weights with adaptive learning rate, cold-start blending,
152
191
  entropy enforcement, and temporal decay. After enough observations, call
153
192
  `mem.best_model("code_generation")` to pick the model that historically
154
193
  performs best on that domain.
155
194
 
195
+ > **Status: experimental.** The engine is tested and works, but discrimination
196
+ > between models only emerges from a *spread* of real outcomes over time — feed
197
+ > it a stream that is mostly successes and every model converges near the cap.
198
+ > Treat `best_model()` as a hint to validate, not a proven router yet. The
199
+ > lesson + known-issue loops above are the proven core (see the A/B benchmark).
200
+
156
201
  ## Semantic retrieval (optional)
157
202
 
158
203
  By default, errlore finds relevant lessons via word overlap (zero
@@ -167,6 +212,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
167
212
  mem = AgentMemory("./agent_memory", embeddings=True)
168
213
  ```
169
214
 
215
+ > The embedding model (~120 MB ONNX) is downloaded once on first use, then
216
+ > runs locally with no further network calls. The core (word-overlap) stays
217
+ > fully offline and dependency-free.
218
+
170
219
  ### Benchmark (adversarial paraphrasing)
171
220
 
172
221
  Tested on 40 lessons with adversarially paraphrased queries
@@ -188,15 +237,37 @@ queries with shared vocabulary, word-overlap works fine.
188
237
  errlore is framework-agnostic. It produces a text block; you put it in the
189
238
  system prompt.
190
239
 
191
- | Provider | Example |
192
- |------------|------------------------------------------------------|
193
- | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
194
- | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
195
- | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
240
+ **Claude Code** — one command wires up failure-memory across sessions:
241
+
242
+ ```bash
243
+ errlore init claude-code # or: --project for this repo only
244
+ ```
196
245
 
197
- All examples run offline with `python examples/<name>.py` (mock responses,
246
+ Failed Bash commands become lessons; every new session is briefed on past
247
+ pitfalls. See [examples/claude-code/](examples/claude-code/).
248
+
249
+ | Provider | Example |
250
+ |-------------|------------------------------------------------------|
251
+ | Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
252
+ | Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
253
+ | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
254
+ | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
255
+ | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
256
+
257
+ The SDK examples run offline with `python examples/<name>.py` (mock responses,
198
258
  no API keys). Set `use_api=True` to call real models.
199
259
 
260
+ ### CLI
261
+
262
+ `pip install errlore` also installs an `errlore` command:
263
+
264
+ ```bash
265
+ errlore init claude-code # install Claude Code hooks + settings
266
+ errlore stats # memory stats for a data dir (--data-dir)
267
+ errlore lessons # list stored lessons
268
+ errlore --version
269
+ ```
270
+
200
271
  ## API overview
201
272
 
202
273
  The main entry point is `AgentMemory`. All other classes are internal --
@@ -210,7 +281,7 @@ you only need them for advanced use.
210
281
  | `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
211
282
  | `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
212
283
  | `lessons(limit)` | List all lessons (sorted by confidence). |
213
- | `best_model(domain)` | Model with the highest trust weight. |
284
+ | `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
214
285
  | `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
215
286
  | `pending_injections()` | Injections not yet reported. |
216
287
  | `stats()` | Aggregate counts + trust weights. |
@@ -236,6 +307,49 @@ you only need them for advanced use.
236
307
  (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
237
308
  (embedding metadata), `trust.json` (trust engine state).
238
309
 
310
+ ## Security
311
+
312
+ A lesson is **trusted prompt content by design** — it is injected into your
313
+ prompts and reaches the model. So:
314
+
315
+ - **Do not ingest lessons from untrusted sources without review.** Treat lesson
316
+ capture like a code review, not like user input. A malicious lesson is a
317
+ prompt-injection vector — and this is the real control, not the sanitizer.
318
+ - **What the sanitizer does (and does not) do.** The lesson *pattern* passes
319
+ `sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
320
+ so log blobs don't pollute the prompt. It is a noise filter, **not** an
321
+ injection defense — it does not neutralize natural-language instructions, and
322
+ the *solution* text is stored as you author it (so it can hold real code).
323
+ Don't rely on it to make untrusted lessons safe.
324
+ - You control what becomes a lesson (`resolve(..., lesson=...)` /
325
+ `add_lesson(...)`); nothing is auto-promoted from raw model output.
326
+
327
+ Report security issues to the address in [SECURITY.md](SECURITY.md).
328
+
329
+ ## Scale & limits (honest)
330
+
331
+ errlore is built for **one process, thousands of lessons** — a single agent or
332
+ a coding-agent session, not a high-throughput fleet. Know the edges:
333
+
334
+ - **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
335
+ ledger each call, so at very high injection volumes it slows down (roughly
336
+ linear in total injections). Fine for interactive/agent use; log compaction
337
+ is the next roadmap item. If you don't need the reinforcement loop, you can
338
+ ignore `report_outcome` and the file stays small.
339
+ - **Single-process by default.** The lesson/error stores use cross-process file
340
+ locks and are safe to share, but the **trust engine and the optional vector
341
+ index are not cross-process safe** — two processes writing `trust.json` /
342
+ `vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
343
+ writer, or give each process its own `data_dir`. Multi-agent shared memory is
344
+ on the roadmap.
345
+ - **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
346
+ over a large existing lesson store is slow the first time (then incremental).
347
+ - Concurrency is tested across threads; **multi-process** stress is not yet in
348
+ the suite.
349
+
350
+ None of these bite at the scale errlore targets today; they're stated so you
351
+ can decide, not discover.
352
+
239
353
  ## Roadmap
240
354
 
241
355
  - [ ] Log compaction for injections journal
@@ -18,14 +18,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
18
18
  into the prompt for similar future tasks.
19
19
  - **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
20
20
  extraction tasks") injected as warnings.
21
- - **Trust** -- Bayesian per-model, per-domain trust weights: know which model to pick
22
- for which job, based on observed outcomes.
21
+ - **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
22
+ point for which model to pick per job, based on observed outcomes. Needs a spread of
23
+ real outcomes to separate models; shipped, but not yet proven on production traffic.
23
24
  - **Closed loop** -- errlore tracks whether an injected lesson actually helped and
24
25
  reinforces or decays it automatically.
25
26
 
26
27
  Embedded, file-based (JSONL), no server, no database, no API keys required.
27
28
  Works fully offline. Your data never leaves your machine.
28
29
 
30
+ ## Who it's for
31
+
32
+ errlore isn't memory for everything — it's memory for **failures**. It shines
33
+ wherever an agent repeats the *same class* of mistake:
34
+
35
+ - **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
36
+ the same bug or forgetting a project convention across sessions.
37
+ - **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
38
+ same date format, rounding rule, or schema field every week.
39
+ - **Any repeated-failure workflow** where a fix should stick the first time,
40
+ not be re-discovered on every run.
41
+
42
+ It fixes what the model doesn't *know* (a convention, a gotcha), not what it
43
+ *can't do* — see the benchmark below.
44
+
29
45
  ## Quickstart (< 5 minutes)
30
46
 
31
47
  ```bash
@@ -65,11 +81,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
65
81
 
66
82
  ## Does it actually reduce errors?
67
83
 
68
- Yes -- for the class of errors memory can fix. Paired A/B benchmark
69
- (`benchmarks/bench_error_reduction.py`): the same model (claude-haiku-4-5)
70
- runs 96 tasks twice, with and without errlore injection. Deterministic
71
- validators, no LLM judges; raw outputs committed in
72
- [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/).
84
+ For the class of errors memory can fix yes, and here's the honest version.
85
+ Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
86
+ (claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
87
+ Deterministic validators, no LLM judges; raw outputs committed in
88
+ [benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
89
+ you can recompute every number.
73
90
 
74
91
  | arm | failures | fail rate |
75
92
  |-----|----------|-----------|
@@ -80,14 +97,36 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
80
97
  Split by error class:
81
98
 
82
99
  - **Knowledge-gap errors** (workspace conventions: date formats, ID
83
- normalization, rounding rules, CSV column order): **46/48 -> 0/48, a 100%
84
- reduction.** The model didn't know the convention; a lesson told it.
100
+ normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
101
+ can't know a convention it was never told, so arm A fails almost by
102
+ construction; the result shows errlore **captures the fix once and re-supplies
103
+ it** on the next similar task, end to end. That store-and-inject loop is the
104
+ claim — not that memory teaches skills.
85
105
  - **Capability-gap errors** (letter counting, string reversal): 17/48 ->
86
106
  20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
87
107
  model doesn't know, not what it can't do.
88
108
 
89
- Reproduce: `python benchmarks/bench_error_reduction.py` (needs an Anthropic
90
- API key; the task families and validators ship in the repo).
109
+ **Reproduced across 5 independent runs** (two on the default seed 5 days apart,
110
+ plus three fresh RNG seeds different task instances). Every run: overall
111
+ reduction **66.7–69.8%**, exact McNemar **p between 8.4e-12 and 1.8e-9**,
112
+ knowledge-gap reduction **95–100%**, capability-gap **−12% to 0%** (no help).
113
+ **Cross-model:** the same grid on **gemma-4-31b** (a different model family)
114
+ lands at **70.0% reduction** (66.7% → 20.0%, p = 2.6e-13), knowledge-gap 83%,
115
+ capability-gap −20% — same effect, same honest boundary. **Task-generality:**
116
+ two fresh realistic-convention families (an arbitrary internal status enum and
117
+ a non-standard git branch convention) go **100% → 0%** on both models.
118
+ Full table + per-run reports:
119
+ [benchmarks/results/REPRODUCIBILITY_2026-07-11.md](benchmarks/results/REPRODUCIBILITY_2026-07-11.md).
120
+
121
+ **Caveats, up front:** temperature 0 still leaves LLM output slightly
122
+ non-deterministic, so exact fine-grained counts vary run to run — the large
123
+ knowledge-gap effect is robust across all five runs; the capability-gap delta
124
+ stays within noise. The knowledge-gap task families use conventions the model
125
+ demonstrably can't guess, which is the point — but it means the headline is
126
+ "the loop works," not "90% fewer errors everywhere."
127
+
128
+ Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
129
+ (needs an Anthropic API key; task families and validators ship in the repo).
91
130
 
92
131
  ## How it works
93
132
 
@@ -112,13 +151,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
112
151
  type before, `inject_for` adds a warning block to the prompt. Separate from
113
152
  lessons: lessons are *solutions*, known issues are *warnings*.
114
153
 
115
- ### 3. Trust loop
154
+ ### 3. Trust loop *(experimental)*
116
155
 
117
156
  Bayesian per-model weights with adaptive learning rate, cold-start blending,
118
157
  entropy enforcement, and temporal decay. After enough observations, call
119
158
  `mem.best_model("code_generation")` to pick the model that historically
120
159
  performs best on that domain.
121
160
 
161
+ > **Status: experimental.** The engine is tested and works, but discrimination
162
+ > between models only emerges from a *spread* of real outcomes over time — feed
163
+ > it a stream that is mostly successes and every model converges near the cap.
164
+ > Treat `best_model()` as a hint to validate, not a proven router yet. The
165
+ > lesson + known-issue loops above are the proven core (see the A/B benchmark).
166
+
122
167
  ## Semantic retrieval (optional)
123
168
 
124
169
  By default, errlore finds relevant lessons via word overlap (zero
@@ -133,6 +178,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
133
178
  mem = AgentMemory("./agent_memory", embeddings=True)
134
179
  ```
135
180
 
181
+ > The embedding model (~120 MB ONNX) is downloaded once on first use, then
182
+ > runs locally with no further network calls. The core (word-overlap) stays
183
+ > fully offline and dependency-free.
184
+
136
185
  ### Benchmark (adversarial paraphrasing)
137
186
 
138
187
  Tested on 40 lessons with adversarially paraphrased queries
@@ -154,15 +203,37 @@ queries with shared vocabulary, word-overlap works fine.
154
203
  errlore is framework-agnostic. It produces a text block; you put it in the
155
204
  system prompt.
156
205
 
157
- | Provider | Example |
158
- |------------|------------------------------------------------------|
159
- | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
160
- | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
161
- | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
206
+ **Claude Code** — one command wires up failure-memory across sessions:
207
+
208
+ ```bash
209
+ errlore init claude-code # or: --project for this repo only
210
+ ```
162
211
 
163
- All examples run offline with `python examples/<name>.py` (mock responses,
212
+ Failed Bash commands become lessons; every new session is briefed on past
213
+ pitfalls. See [examples/claude-code/](examples/claude-code/).
214
+
215
+ | Provider | Example |
216
+ |-------------|------------------------------------------------------|
217
+ | Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
218
+ | Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
219
+ | OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
220
+ | Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
221
+ | LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
222
+
223
+ The SDK examples run offline with `python examples/<name>.py` (mock responses,
164
224
  no API keys). Set `use_api=True` to call real models.
165
225
 
226
+ ### CLI
227
+
228
+ `pip install errlore` also installs an `errlore` command:
229
+
230
+ ```bash
231
+ errlore init claude-code # install Claude Code hooks + settings
232
+ errlore stats # memory stats for a data dir (--data-dir)
233
+ errlore lessons # list stored lessons
234
+ errlore --version
235
+ ```
236
+
166
237
  ## API overview
167
238
 
168
239
  The main entry point is `AgentMemory`. All other classes are internal --
@@ -176,7 +247,7 @@ you only need them for advanced use.
176
247
  | `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
177
248
  | `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
178
249
  | `lessons(limit)` | List all lessons (sorted by confidence). |
179
- | `best_model(domain)` | Model with the highest trust weight. |
250
+ | `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
180
251
  | `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
181
252
  | `pending_injections()` | Injections not yet reported. |
182
253
  | `stats()` | Aggregate counts + trust weights. |
@@ -202,6 +273,49 @@ you only need them for advanced use.
202
273
  (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
203
274
  (embedding metadata), `trust.json` (trust engine state).
204
275
 
276
+ ## Security
277
+
278
+ A lesson is **trusted prompt content by design** — it is injected into your
279
+ prompts and reaches the model. So:
280
+
281
+ - **Do not ingest lessons from untrusted sources without review.** Treat lesson
282
+ capture like a code review, not like user input. A malicious lesson is a
283
+ prompt-injection vector — and this is the real control, not the sanitizer.
284
+ - **What the sanitizer does (and does not) do.** The lesson *pattern* passes
285
+ `sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
286
+ so log blobs don't pollute the prompt. It is a noise filter, **not** an
287
+ injection defense — it does not neutralize natural-language instructions, and
288
+ the *solution* text is stored as you author it (so it can hold real code).
289
+ Don't rely on it to make untrusted lessons safe.
290
+ - You control what becomes a lesson (`resolve(..., lesson=...)` /
291
+ `add_lesson(...)`); nothing is auto-promoted from raw model output.
292
+
293
+ Report security issues to the address in [SECURITY.md](SECURITY.md).
294
+
295
+ ## Scale & limits (honest)
296
+
297
+ errlore is built for **one process, thousands of lessons** — a single agent or
298
+ a coding-agent session, not a high-throughput fleet. Know the edges:
299
+
300
+ - **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
301
+ ledger each call, so at very high injection volumes it slows down (roughly
302
+ linear in total injections). Fine for interactive/agent use; log compaction
303
+ is the next roadmap item. If you don't need the reinforcement loop, you can
304
+ ignore `report_outcome` and the file stays small.
305
+ - **Single-process by default.** The lesson/error stores use cross-process file
306
+ locks and are safe to share, but the **trust engine and the optional vector
307
+ index are not cross-process safe** — two processes writing `trust.json` /
308
+ `vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
309
+ writer, or give each process its own `data_dir`. Multi-agent shared memory is
310
+ on the roadmap.
311
+ - **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
312
+ over a large existing lesson store is slow the first time (then incremental).
313
+ - Concurrency is tested across threads; **multi-process** stress is not yet in
314
+ the suite.
315
+
316
+ None of these bite at the scale errlore targets today; they're stated so you
317
+ can decide, not discover.
318
+
205
319
  ## Roadmap
206
320
 
207
321
  - [ ] Log compaction for injections journal