errlore 0.1.3__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- errlore-0.2.0/.github/workflows/release.yml +74 -0
- {errlore-0.1.3 → errlore-0.2.0}/CHANGELOG.md +54 -0
- {errlore-0.1.3 → errlore-0.2.0}/PKG-INFO +135 -21
- {errlore-0.1.3 → errlore-0.2.0}/README.md +133 -19
- {errlore-0.1.3 → errlore-0.2.0}/benchmarks/bench_error_reduction.py +60 -2
- errlore-0.2.0/benchmarks/results/REPRODUCIBILITY_2026-07-11.md +105 -0
- errlore-0.2.0/benchmarks/results/gemma_2026-07-11_report.txt +37 -0
- errlore-0.2.0/benchmarks/results/haiku_2026-07-11_report.txt +35 -0
- errlore-0.2.0/benchmarks/results/haiku_2026-07-11_seed11.txt +35 -0
- errlore-0.2.0/benchmarks/results/haiku_2026-07-11_seed22.txt +35 -0
- errlore-0.2.0/benchmarks/results/haiku_2026-07-11_seed33.txt +35 -0
- errlore-0.2.0/benchmarks/results/haiku_newfamilies_2026-07-11_report.txt +27 -0
- {errlore-0.1.3 → errlore-0.2.0}/examples/anthropic_agent.py +4 -4
- errlore-0.2.0/examples/claude-code/README.md +37 -0
- {errlore-0.1.3 → errlore-0.2.0}/examples/langchain_agent.py +4 -3
- {errlore-0.1.3 → errlore-0.2.0}/examples/openai_agent.py +4 -3
- {errlore-0.1.3 → errlore-0.2.0}/pyproject.toml +5 -2
- {errlore-0.1.3 → errlore-0.2.0}/site/index.html +99 -26
- {errlore-0.1.3 → errlore-0.2.0}/site/llms.txt +9 -6
- errlore-0.2.0/site/robots.txt +12 -0
- errlore-0.2.0/site/sitemap.xml +9 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/__init__.py +1 -1
- errlore-0.2.0/src/errlore/cli.py +200 -0
- errlore-0.2.0/src/errlore/integrations/__init__.py +5 -0
- errlore-0.2.0/src/errlore/integrations/claude_code.py +92 -0
- errlore-0.2.0/tests/test_cli.py +145 -0
- errlore-0.1.3/examples/claude-code/README.md +0 -23
- {errlore-0.1.3 → errlore-0.2.0}/.github/workflows/ci.yml +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/.gitignore +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/LICENSE +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/SECURITY.md +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/benchmarks/bench_retrieval.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/benchmarks/results/error_reduction/report.md +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/examples/claude-code/errlore_posttooluse.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/examples/claude-code/errlore_sessionstart.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/examples/claude-code/settings.json.example +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/integrations/openwebui/README.md +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/integrations/openwebui/errlore_feedback_action.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/integrations/openwebui/errlore_memory_filter.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/site/demo.gif +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/site/demo_script.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/site/og.png +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/classifier.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/injector.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/patterns.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/errmem/tracker.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/facade.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/jsonl_index.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/jsonl_writer.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/io/repair.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/lessons/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/lessons/models.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/lessons/store.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/py.typed +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/retrieval/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/retrieval/backend.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/retrieval/index.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/sanitize.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/trust/__init__.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/src/errlore/trust/engine.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/conftest.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_errmem.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_facade.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_io.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_lessons.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_openwebui_integration.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_regressions.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_retrieval.py +0 -0
- {errlore-0.1.3 → errlore-0.2.0}/tests/test_trust.py +0 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
# Tag-driven release: push a v* tag -> full test gate -> build sdist+wheel ->
|
|
4
|
+
# publish to PyPI via trusted publishing (OIDC; no long-lived token in CI).
|
|
5
|
+
#
|
|
6
|
+
# One-time PyPI setup (owner): pypi.org -> project errlore -> Publishing ->
|
|
7
|
+
# add GitHub publisher (owner=Ma4etaSS, repo=errlore, workflow=release.yml,
|
|
8
|
+
# environment=pypi). Until that's configured the publish step fails cleanly
|
|
9
|
+
# and the build artifacts are still attached to the GitHub release.
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
push:
|
|
13
|
+
tags: ["v*"]
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
test:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: "3.12"
|
|
23
|
+
- run: pip install -e .[dev]
|
|
24
|
+
- run: ruff check .
|
|
25
|
+
- run: mypy
|
|
26
|
+
- run: pytest --cov=errlore --cov-fail-under=80
|
|
27
|
+
|
|
28
|
+
build:
|
|
29
|
+
needs: test
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v4
|
|
33
|
+
- uses: actions/setup-python@v5
|
|
34
|
+
with:
|
|
35
|
+
python-version: "3.12"
|
|
36
|
+
- run: pip install build
|
|
37
|
+
- run: python -m build
|
|
38
|
+
- uses: actions/upload-artifact@v4
|
|
39
|
+
with:
|
|
40
|
+
name: dist
|
|
41
|
+
path: dist/
|
|
42
|
+
|
|
43
|
+
publish:
|
|
44
|
+
needs: build
|
|
45
|
+
runs-on: ubuntu-latest
|
|
46
|
+
environment: pypi
|
|
47
|
+
permissions:
|
|
48
|
+
id-token: write # OIDC for PyPI trusted publishing
|
|
49
|
+
steps:
|
|
50
|
+
- uses: actions/download-artifact@v4
|
|
51
|
+
with:
|
|
52
|
+
name: dist
|
|
53
|
+
path: dist/
|
|
54
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
55
|
+
|
|
56
|
+
github-release:
|
|
57
|
+
needs: build
|
|
58
|
+
runs-on: ubuntu-latest
|
|
59
|
+
permissions:
|
|
60
|
+
contents: write
|
|
61
|
+
steps:
|
|
62
|
+
- uses: actions/checkout@v4
|
|
63
|
+
- uses: actions/download-artifact@v4
|
|
64
|
+
with:
|
|
65
|
+
name: dist
|
|
66
|
+
path: dist/
|
|
67
|
+
- name: Create GitHub release with artifacts
|
|
68
|
+
env:
|
|
69
|
+
GH_TOKEN: ${{ github.token }}
|
|
70
|
+
run: |
|
|
71
|
+
gh release create "${GITHUB_REF_NAME}" dist/* \
|
|
72
|
+
--title "${GITHUB_REF_NAME}" \
|
|
73
|
+
--notes "See CHANGELOG.md for details." || \
|
|
74
|
+
gh release upload "${GITHUB_REF_NAME}" dist/* --clobber
|
|
@@ -5,6 +5,60 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.2.0] - 2026-07-11 — "The Proof Release"
|
|
9
|
+
|
|
10
|
+
The headline benchmark is now proven on all three generality axes:
|
|
11
|
+
**seed-robustness** (5 independent runs), **task-generality** (fresh realistic
|
|
12
|
+
convention families), and **model-diversity** (Anthropic + Gemma). Development
|
|
13
|
+
status bumped Alpha → Beta.
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- **Reproducibility evidence for the error-reduction benchmark**
|
|
17
|
+
(`benchmarks/results/REPRODUCIBILITY_2026-07-11.md`): the headline A/B is now
|
|
18
|
+
reproduced across 5 independent runs on claude-haiku-4-5 (two default-seed
|
|
19
|
+
runs 5 days apart + three fresh RNG seeds). Every run: 66.7–69.8% repeat-error
|
|
20
|
+
reduction, exact McNemar p between 8.4e-12 and 1.8e-9, knowledge-gap reduction
|
|
21
|
+
95–100%, capability-gap −12% to 0%. Resolves the prior "single run" caveat in
|
|
22
|
+
the README.
|
|
23
|
+
- `bench_error_reduction.py` reads `BENCH_RNG_SEED` from the environment so the
|
|
24
|
+
same task families can be re-drawn with different instances for seed-robustness
|
|
25
|
+
checks. Also `BENCH_FAMILIES` / `BENCH_SEED_N` / `BENCH_TEST_N` for narrowing a
|
|
26
|
+
run (e.g. a small cross-family probe on a tight free-tier quota).
|
|
27
|
+
- **Task-generality: two new realistic knowledge-gap families** (`status_code` —
|
|
28
|
+
an arbitrary internal status enum; `branch_name` — a non-standard git branch
|
|
29
|
+
convention). On claude-haiku-4-5 both go 100% → 0% (24/24 → 0/24, McNemar
|
|
30
|
+
p=1.19e-07), confirming the store-and-inject effect isn't specific to the
|
|
31
|
+
original toy families. Report + raw outputs in `benchmarks/results/`.
|
|
32
|
+
- **Model-diversity: full grid on gemma-4-31b** (Cerebras — a different model
|
|
33
|
+
family): 66.7% → 20.0% fail (70.0% reduction, McNemar p=2.6e-13),
|
|
34
|
+
knowledge-gap 83%, capability-gap −20%; branch_name and status_code flip
|
|
35
|
+
100%→0% here too. One honest wrinkle documented: `csv_order` does not
|
|
36
|
+
transfer to gemma (12→12) while it flips to 0 on Haiku — lesson-following is
|
|
37
|
+
itself model-dependent at the margin. `BENCH_MODEL` env override added for
|
|
38
|
+
running any backend against a specific model.
|
|
39
|
+
|
|
40
|
+
## [0.1.4] - 2026-07-09
|
|
41
|
+
|
|
42
|
+
### Added
|
|
43
|
+
- **CLI** (`errlore` console command): `errlore init claude-code` writes the
|
|
44
|
+
two Claude Code hook scripts and idempotently merges them into your
|
|
45
|
+
`settings.json` (global or `--project`), preserving existing hooks — a
|
|
46
|
+
one-command install instead of copy/edit/merge. Plus `errlore stats` and
|
|
47
|
+
`errlore lessons`.
|
|
48
|
+
- `errlore.integrations.claude_code` — the hook logic (`post_tool_use`,
|
|
49
|
+
`session_start`) now ships in the package and is tested, so the generated
|
|
50
|
+
hooks are 3-line shims.
|
|
51
|
+
|
|
52
|
+
### Changed
|
|
53
|
+
- README: honest A/B framing (the knowledge-gap baseline fails by
|
|
54
|
+
construction; the result shows the capture-and-re-supply loop works, not
|
|
55
|
+
that memory teaches skills; single-run-at-temp-0 caveat). Coding-agent-first
|
|
56
|
+
hero. Security section reworded — the sanitizer is a noise filter on the
|
|
57
|
+
pattern, not an injection defense. Added a "Scale & limits" section
|
|
58
|
+
(unbounded injections journal, single-process trust/vector index).
|
|
59
|
+
- Examples no longer hard-code aging frontier model ids; they use
|
|
60
|
+
`os.getenv(...)` with a small default.
|
|
61
|
+
|
|
8
62
|
## [0.1.3] - 2026-07-06
|
|
9
63
|
|
|
10
64
|
### Fixed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: errlore
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server.
|
|
5
5
|
Project-URL: Homepage, https://errlore.com
|
|
6
6
|
Project-URL: Repository, https://github.com/Ma4etaSS/errlore
|
|
@@ -10,7 +10,7 @@ Author: Ma4etaSS
|
|
|
10
10
|
License-Expression: MIT
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Keywords: agents,error-learning,lessons,llm,memory,trust
|
|
13
|
-
Classifier: Development Status ::
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
14
|
Classifier: Intended Audience :: Developers
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.10
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.11
|
|
@@ -52,14 +52,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
|
|
|
52
52
|
into the prompt for similar future tasks.
|
|
53
53
|
- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
|
|
54
54
|
extraction tasks") injected as warnings.
|
|
55
|
-
- **Trust** -- Bayesian per-model, per-domain trust weights:
|
|
56
|
-
for which job, based on observed outcomes.
|
|
55
|
+
- **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
|
|
56
|
+
point for which model to pick per job, based on observed outcomes. Needs a spread of
|
|
57
|
+
real outcomes to separate models; shipped, but not yet proven on production traffic.
|
|
57
58
|
- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
|
|
58
59
|
reinforces or decays it automatically.
|
|
59
60
|
|
|
60
61
|
Embedded, file-based (JSONL), no server, no database, no API keys required.
|
|
61
62
|
Works fully offline. Your data never leaves your machine.
|
|
62
63
|
|
|
64
|
+
## Who it's for
|
|
65
|
+
|
|
66
|
+
errlore isn't memory for everything — it's memory for **failures**. It shines
|
|
67
|
+
wherever an agent repeats the *same class* of mistake:
|
|
68
|
+
|
|
69
|
+
- **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
|
|
70
|
+
the same bug or forgetting a project convention across sessions.
|
|
71
|
+
- **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
|
|
72
|
+
same date format, rounding rule, or schema field every week.
|
|
73
|
+
- **Any repeated-failure workflow** where a fix should stick the first time,
|
|
74
|
+
not be re-discovered on every run.
|
|
75
|
+
|
|
76
|
+
It fixes what the model doesn't *know* (a convention, a gotcha), not what it
|
|
77
|
+
*can't do* — see the benchmark below.
|
|
78
|
+
|
|
63
79
|
## Quickstart (< 5 minutes)
|
|
64
80
|
|
|
65
81
|
```bash
|
|
@@ -99,11 +115,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
|
|
|
99
115
|
|
|
100
116
|
## Does it actually reduce errors?
|
|
101
117
|
|
|
102
|
-
|
|
103
|
-
(`benchmarks/bench_error_reduction.py`): the same model
|
|
104
|
-
runs 96 tasks twice, with and without errlore injection.
|
|
105
|
-
validators, no LLM judges; raw outputs committed in
|
|
106
|
-
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/)
|
|
118
|
+
For the class of errors memory can fix — yes, and here's the honest version.
|
|
119
|
+
Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
|
|
120
|
+
(claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
|
|
121
|
+
Deterministic validators, no LLM judges; raw outputs committed in
|
|
122
|
+
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
|
|
123
|
+
you can recompute every number.
|
|
107
124
|
|
|
108
125
|
| arm | failures | fail rate |
|
|
109
126
|
|-----|----------|-----------|
|
|
@@ -114,14 +131,36 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
|
|
|
114
131
|
Split by error class:
|
|
115
132
|
|
|
116
133
|
- **Knowledge-gap errors** (workspace conventions: date formats, ID
|
|
117
|
-
normalization, rounding rules, CSV column order):
|
|
118
|
-
|
|
134
|
+
normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
|
|
135
|
+
can't know a convention it was never told, so arm A fails almost by
|
|
136
|
+
construction; the result shows errlore **captures the fix once and re-supplies
|
|
137
|
+
it** on the next similar task, end to end. That store-and-inject loop is the
|
|
138
|
+
claim — not that memory teaches skills.
|
|
119
139
|
- **Capability-gap errors** (letter counting, string reversal): 17/48 ->
|
|
120
140
|
20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
|
|
121
141
|
model doesn't know, not what it can't do.
|
|
122
142
|
|
|
123
|
-
|
|
124
|
-
|
|
143
|
+
**Reproduced across 5 independent runs** (two on the default seed 5 days apart,
|
|
144
|
+
plus three fresh RNG seeds — different task instances). Every run: overall
|
|
145
|
+
reduction **66.7–69.8%**, exact McNemar **p between 8.4e-12 and 1.8e-9**,
|
|
146
|
+
knowledge-gap reduction **95–100%**, capability-gap **−12% to 0%** (no help).
|
|
147
|
+
**Cross-model:** the same grid on **gemma-4-31b** (a different model family)
|
|
148
|
+
lands at **70.0% reduction** (66.7% → 20.0%, p = 2.6e-13), knowledge-gap 83%,
|
|
149
|
+
capability-gap −20% — same effect, same honest boundary. **Task-generality:**
|
|
150
|
+
two fresh realistic-convention families (an arbitrary internal status enum and
|
|
151
|
+
a non-standard git branch convention) go **100% → 0%** on both models.
|
|
152
|
+
Full table + per-run reports:
|
|
153
|
+
[benchmarks/results/REPRODUCIBILITY_2026-07-11.md](benchmarks/results/REPRODUCIBILITY_2026-07-11.md).
|
|
154
|
+
|
|
155
|
+
**Caveats, up front:** temperature 0 still leaves LLM output slightly
|
|
156
|
+
non-deterministic, so exact fine-grained counts vary run to run — the large
|
|
157
|
+
knowledge-gap effect is robust across all five runs; the capability-gap delta
|
|
158
|
+
stays within noise. The knowledge-gap task families use conventions the model
|
|
159
|
+
demonstrably can't guess, which is the point — but it means the headline is
|
|
160
|
+
"the loop works," not "90% fewer errors everywhere."
|
|
161
|
+
|
|
162
|
+
Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
|
|
163
|
+
(needs an Anthropic API key; task families and validators ship in the repo).
|
|
125
164
|
|
|
126
165
|
## How it works
|
|
127
166
|
|
|
@@ -146,13 +185,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
|
|
|
146
185
|
type before, `inject_for` adds a warning block to the prompt. Separate from
|
|
147
186
|
lessons: lessons are *solutions*, known issues are *warnings*.
|
|
148
187
|
|
|
149
|
-
### 3. Trust loop
|
|
188
|
+
### 3. Trust loop *(experimental)*
|
|
150
189
|
|
|
151
190
|
Bayesian per-model weights with adaptive learning rate, cold-start blending,
|
|
152
191
|
entropy enforcement, and temporal decay. After enough observations, call
|
|
153
192
|
`mem.best_model("code_generation")` to pick the model that historically
|
|
154
193
|
performs best on that domain.
|
|
155
194
|
|
|
195
|
+
> **Status: experimental.** The engine is tested and works, but discrimination
|
|
196
|
+
> between models only emerges from a *spread* of real outcomes over time — feed
|
|
197
|
+
> it a stream that is mostly successes and every model converges near the cap.
|
|
198
|
+
> Treat `best_model()` as a hint to validate, not a proven router yet. The
|
|
199
|
+
> lesson + known-issue loops above are the proven core (see the A/B benchmark).
|
|
200
|
+
|
|
156
201
|
## Semantic retrieval (optional)
|
|
157
202
|
|
|
158
203
|
By default, errlore finds relevant lessons via word overlap (zero
|
|
@@ -167,6 +212,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
|
|
|
167
212
|
mem = AgentMemory("./agent_memory", embeddings=True)
|
|
168
213
|
```
|
|
169
214
|
|
|
215
|
+
> The embedding model (~120 MB ONNX) is downloaded once on first use, then
|
|
216
|
+
> runs locally with no further network calls. The core (word-overlap) stays
|
|
217
|
+
> fully offline and dependency-free.
|
|
218
|
+
|
|
170
219
|
### Benchmark (adversarial paraphrasing)
|
|
171
220
|
|
|
172
221
|
Tested on 40 lessons with adversarially paraphrased queries
|
|
@@ -188,15 +237,37 @@ queries with shared vocabulary, word-overlap works fine.
|
|
|
188
237
|
errlore is framework-agnostic. It produces a text block; you put it in the
|
|
189
238
|
system prompt.
|
|
190
239
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
240
|
+
**Claude Code** — one command wires up failure-memory across sessions:
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
errlore init claude-code # or: --project for this repo only
|
|
244
|
+
```
|
|
196
245
|
|
|
197
|
-
|
|
246
|
+
Failed Bash commands become lessons; every new session is briefed on past
|
|
247
|
+
pitfalls. See [examples/claude-code/](examples/claude-code/).
|
|
248
|
+
|
|
249
|
+
| Provider | Example |
|
|
250
|
+
|-------------|------------------------------------------------------|
|
|
251
|
+
| Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
|
|
252
|
+
| Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
|
|
253
|
+
| OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
|
|
254
|
+
| Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
|
|
255
|
+
| LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
|
|
256
|
+
|
|
257
|
+
The SDK examples run offline with `python examples/<name>.py` (mock responses,
|
|
198
258
|
no API keys). Set `use_api=True` to call real models.
|
|
199
259
|
|
|
260
|
+
### CLI
|
|
261
|
+
|
|
262
|
+
`pip install errlore` also installs an `errlore` command:
|
|
263
|
+
|
|
264
|
+
```bash
|
|
265
|
+
errlore init claude-code # install Claude Code hooks + settings
|
|
266
|
+
errlore stats # memory stats for a data dir (--data-dir)
|
|
267
|
+
errlore lessons # list stored lessons
|
|
268
|
+
errlore --version
|
|
269
|
+
```
|
|
270
|
+
|
|
200
271
|
## API overview
|
|
201
272
|
|
|
202
273
|
The main entry point is `AgentMemory`. All other classes are internal --
|
|
@@ -210,7 +281,7 @@ you only need them for advanced use.
|
|
|
210
281
|
| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
|
|
211
282
|
| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
|
|
212
283
|
| `lessons(limit)` | List all lessons (sorted by confidence). |
|
|
213
|
-
| `best_model(domain)` | Model with the highest trust weight
|
|
284
|
+
| `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
|
|
214
285
|
| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
|
|
215
286
|
| `pending_injections()` | Injections not yet reported. |
|
|
216
287
|
| `stats()` | Aggregate counts + trust weights. |
|
|
@@ -236,6 +307,49 @@ you only need them for advanced use.
|
|
|
236
307
|
(filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
|
|
237
308
|
(embedding metadata), `trust.json` (trust engine state).
|
|
238
309
|
|
|
310
|
+
## Security
|
|
311
|
+
|
|
312
|
+
A lesson is **trusted prompt content by design** — it is injected into your
|
|
313
|
+
prompts and reaches the model. So:
|
|
314
|
+
|
|
315
|
+
- **Do not ingest lessons from untrusted sources without review.** Treat lesson
|
|
316
|
+
capture like a code review, not like user input. A malicious lesson is a
|
|
317
|
+
prompt-injection vector — and this is the real control, not the sanitizer.
|
|
318
|
+
- **What the sanitizer does (and does not) do.** The lesson *pattern* passes
|
|
319
|
+
`sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
|
|
320
|
+
so log blobs don't pollute the prompt. It is a noise filter, **not** an
|
|
321
|
+
injection defense — it does not neutralize natural-language instructions, and
|
|
322
|
+
the *solution* text is stored as you author it (so it can hold real code).
|
|
323
|
+
Don't rely on it to make untrusted lessons safe.
|
|
324
|
+
- You control what becomes a lesson (`resolve(..., lesson=...)` /
|
|
325
|
+
`add_lesson(...)`); nothing is auto-promoted from raw model output.
|
|
326
|
+
|
|
327
|
+
Report security issues to the address in [SECURITY.md](SECURITY.md).
|
|
328
|
+
|
|
329
|
+
## Scale & limits (honest)
|
|
330
|
+
|
|
331
|
+
errlore is built for **one process, thousands of lessons** — a single agent or
|
|
332
|
+
a coding-agent session, not a high-throughput fleet. Know the edges:
|
|
333
|
+
|
|
334
|
+
- **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
|
|
335
|
+
ledger each call, so at very high injection volumes it slows down (roughly
|
|
336
|
+
linear in total injections). Fine for interactive/agent use; log compaction
|
|
337
|
+
is the next roadmap item. If you don't need the reinforcement loop, you can
|
|
338
|
+
ignore `report_outcome` and the file stays small.
|
|
339
|
+
- **Single-process by default.** The lesson/error stores use cross-process file
|
|
340
|
+
locks and are safe to share, but the **trust engine and the optional vector
|
|
341
|
+
index are not cross-process safe** — two processes writing `trust.json` /
|
|
342
|
+
`vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
|
|
343
|
+
writer, or give each process its own `data_dir`. Multi-agent shared memory is
|
|
344
|
+
on the roadmap.
|
|
345
|
+
- **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
|
|
346
|
+
over a large existing lesson store is slow the first time (then incremental).
|
|
347
|
+
- Concurrency is tested across threads; **multi-process** stress is not yet in
|
|
348
|
+
the suite.
|
|
349
|
+
|
|
350
|
+
None of these bite at the scale errlore targets today; they're stated so you
|
|
351
|
+
can decide, not discover.
|
|
352
|
+
|
|
239
353
|
## Roadmap
|
|
240
354
|
|
|
241
355
|
- [ ] Log compaction for injections journal
|
|
@@ -18,14 +18,30 @@ Your agent keeps making the same mistakes. errlore fixes that:
|
|
|
18
18
|
into the prompt for similar future tasks.
|
|
19
19
|
- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
|
|
20
20
|
extraction tasks") injected as warnings.
|
|
21
|
-
- **Trust** -- Bayesian per-model, per-domain trust weights:
|
|
22
|
-
for which job, based on observed outcomes.
|
|
21
|
+
- **Trust** *(experimental)* -- Bayesian per-model, per-domain trust weights: a starting
|
|
22
|
+
point for which model to pick per job, based on observed outcomes. Needs a spread of
|
|
23
|
+
real outcomes to separate models; shipped, but not yet proven on production traffic.
|
|
23
24
|
- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
|
|
24
25
|
reinforces or decays it automatically.
|
|
25
26
|
|
|
26
27
|
Embedded, file-based (JSONL), no server, no database, no API keys required.
|
|
27
28
|
Works fully offline. Your data never leaves your machine.
|
|
28
29
|
|
|
30
|
+
## Who it's for
|
|
31
|
+
|
|
32
|
+
errlore isn't memory for everything — it's memory for **failures**. It shines
|
|
33
|
+
wherever an agent repeats the *same class* of mistake:
|
|
34
|
+
|
|
35
|
+
- **Coding agents** (Claude Code, Cursor, SWE agents) that keep re-introducing
|
|
36
|
+
the same bug or forgetting a project convention across sessions.
|
|
37
|
+
- **Extraction pipelines** (PDFs, invoices, contracts) that hallucinate the
|
|
38
|
+
same date format, rounding rule, or schema field every week.
|
|
39
|
+
- **Any repeated-failure workflow** where a fix should stick the first time,
|
|
40
|
+
not be re-discovered on every run.
|
|
41
|
+
|
|
42
|
+
It fixes what the model doesn't *know* (a convention, a gotcha), not what it
|
|
43
|
+
*can't do* — see the benchmark below.
|
|
44
|
+
|
|
29
45
|
## Quickstart (< 5 minutes)
|
|
30
46
|
|
|
31
47
|
```bash
|
|
@@ -65,11 +81,12 @@ JSONL files and does text matching. LLM calls are yours to make (or not).
|
|
|
65
81
|
|
|
66
82
|
## Does it actually reduce errors?
|
|
67
83
|
|
|
68
|
-
|
|
69
|
-
(`benchmarks/bench_error_reduction.py`): the same model
|
|
70
|
-
runs 96 tasks twice, with and without errlore injection.
|
|
71
|
-
validators, no LLM judges; raw outputs committed in
|
|
72
|
-
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/)
|
|
84
|
+
For the class of errors memory can fix — yes, and here's the honest version.
|
|
85
|
+
Paired A/B (`benchmarks/bench_error_reduction.py`): the same model
|
|
86
|
+
(claude-haiku-4-5) runs 96 tasks twice, with and without errlore injection.
|
|
87
|
+
Deterministic validators, no LLM judges; raw outputs committed in
|
|
88
|
+
[benchmarks/results/error_reduction/](benchmarks/results/error_reduction/) so
|
|
89
|
+
you can recompute every number.
|
|
73
90
|
|
|
74
91
|
| arm | failures | fail rate |
|
|
75
92
|
|-----|----------|-----------|
|
|
@@ -80,14 +97,36 @@ Exact McNemar over all 96 pairs: p = 1.8e-09 (49 pairs fixed, 6 broken).
|
|
|
80
97
|
Split by error class:
|
|
81
98
|
|
|
82
99
|
- **Knowledge-gap errors** (workspace conventions: date formats, ID
|
|
83
|
-
normalization, rounding rules, CSV column order):
|
|
84
|
-
|
|
100
|
+
normalization, rounding rules, CSV column order): 46/48 -> 0/48. The model
|
|
101
|
+
can't know a convention it was never told, so arm A fails almost by
|
|
102
|
+
construction; the result shows errlore **captures the fix once and re-supplies
|
|
103
|
+
it** on the next similar task, end to end. That store-and-inject loop is the
|
|
104
|
+
claim — not that memory teaches skills.
|
|
85
105
|
- **Capability-gap errors** (letter counting, string reversal): 17/48 ->
|
|
86
106
|
20/48 -- errlore did **not** help and slightly hurt. Memory fixes what the
|
|
87
107
|
model doesn't know, not what it can't do.
|
|
88
108
|
|
|
89
|
-
|
|
90
|
-
|
|
109
|
+
**Reproduced across 5 independent runs** (two on the default seed 5 days apart,
|
|
110
|
+
plus three fresh RNG seeds — different task instances). Every run: overall
|
|
111
|
+
reduction **66.7–69.8%**, exact McNemar **p between 8.4e-12 and 1.8e-9**,
|
|
112
|
+
knowledge-gap reduction **95–100%**, capability-gap **−12% to 0%** (no help).
|
|
113
|
+
**Cross-model:** the same grid on **gemma-4-31b** (a different model family)
|
|
114
|
+
lands at **70.0% reduction** (66.7% → 20.0%, p = 2.6e-13), knowledge-gap 83%,
|
|
115
|
+
capability-gap −20% — same effect, same honest boundary. **Task-generality:**
|
|
116
|
+
two fresh realistic-convention families (an arbitrary internal status enum and
|
|
117
|
+
a non-standard git branch convention) go **100% → 0%** on both models.
|
|
118
|
+
Full table + per-run reports:
|
|
119
|
+
[benchmarks/results/REPRODUCIBILITY_2026-07-11.md](benchmarks/results/REPRODUCIBILITY_2026-07-11.md).
|
|
120
|
+
|
|
121
|
+
**Caveats, up front:** temperature 0 still leaves LLM output slightly
|
|
122
|
+
non-deterministic, so exact fine-grained counts vary run to run — the large
|
|
123
|
+
knowledge-gap effect is robust across all five runs; the capability-gap delta
|
|
124
|
+
stays within noise. The knowledge-gap task families use conventions the model
|
|
125
|
+
demonstrably can't guess, which is the point — but it means the headline is
|
|
126
|
+
"the loop works," not "90% fewer errors everywhere."
|
|
127
|
+
|
|
128
|
+
Reproduce: `python benchmarks/bench_error_reduction.py --backend anthropic`
|
|
129
|
+
(needs an Anthropic API key; task families and validators ship in the repo).
|
|
91
130
|
|
|
92
131
|
## How it works
|
|
93
132
|
|
|
@@ -112,13 +151,19 @@ Per-model, per-task-type error tracking. When a model has failed on a task
|
|
|
112
151
|
type before, `inject_for` adds a warning block to the prompt. Separate from
|
|
113
152
|
lessons: lessons are *solutions*, known issues are *warnings*.
|
|
114
153
|
|
|
115
|
-
### 3. Trust loop
|
|
154
|
+
### 3. Trust loop *(experimental)*
|
|
116
155
|
|
|
117
156
|
Bayesian per-model weights with adaptive learning rate, cold-start blending,
|
|
118
157
|
entropy enforcement, and temporal decay. After enough observations, call
|
|
119
158
|
`mem.best_model("code_generation")` to pick the model that historically
|
|
120
159
|
performs best on that domain.
|
|
121
160
|
|
|
161
|
+
> **Status: experimental.** The engine is tested and works, but discrimination
|
|
162
|
+
> between models only emerges from a *spread* of real outcomes over time — feed
|
|
163
|
+
> it a stream that is mostly successes and every model converges near the cap.
|
|
164
|
+
> Treat `best_model()` as a hint to validate, not a proven router yet. The
|
|
165
|
+
> lesson + known-issue loops above are the proven core (see the A/B benchmark).
|
|
166
|
+
|
|
122
167
|
## Semantic retrieval (optional)
|
|
123
168
|
|
|
124
169
|
By default, errlore finds relevant lessons via word overlap (zero
|
|
@@ -133,6 +178,10 @@ pip install errlore[embeddings] # installs fastembed + numpy
|
|
|
133
178
|
mem = AgentMemory("./agent_memory", embeddings=True)
|
|
134
179
|
```
|
|
135
180
|
|
|
181
|
+
> The embedding model (~120 MB ONNX) is downloaded once on first use, then
|
|
182
|
+
> runs locally with no further network calls. The core (word-overlap) stays
|
|
183
|
+
> fully offline and dependency-free.
|
|
184
|
+
|
|
136
185
|
### Benchmark (adversarial paraphrasing)
|
|
137
186
|
|
|
138
187
|
Tested on 40 lessons with adversarially paraphrased queries
|
|
@@ -154,15 +203,37 @@ queries with shared vocabulary, word-overlap works fine.
|
|
|
154
203
|
errlore is framework-agnostic. It produces a text block; you put it in the
|
|
155
204
|
system prompt.
|
|
156
205
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
206
|
+
**Claude Code** — one command wires up failure-memory across sessions:
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
errlore init claude-code # or: --project for this repo only
|
|
210
|
+
```
|
|
162
211
|
|
|
163
|
-
|
|
212
|
+
Failed Bash commands become lessons; every new session is briefed on past
|
|
213
|
+
pitfalls. See [examples/claude-code/](examples/claude-code/).
|
|
214
|
+
|
|
215
|
+
| Provider | Example |
|
|
216
|
+
|-------------|------------------------------------------------------|
|
|
217
|
+
| Claude Code | [examples/claude-code/](examples/claude-code/) — hooks, `errlore init claude-code` |
|
|
218
|
+
| Open WebUI | [integrations/openwebui/](integrations/openwebui/) — memory Filter + feedback Action |
|
|
219
|
+
| OpenAI | [examples/openai_agent.py](examples/openai_agent.py) |
|
|
220
|
+
| Anthropic | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
|
|
221
|
+
| LangChain | [examples/langchain_agent.py](examples/langchain_agent.py) |
|
|
222
|
+
|
|
223
|
+
The SDK examples run offline with `python examples/<name>.py` (mock responses,
|
|
164
224
|
no API keys). Set `use_api=True` to call real models.
|
|
165
225
|
|
|
226
|
+
### CLI
|
|
227
|
+
|
|
228
|
+
`pip install errlore` also installs an `errlore` command:
|
|
229
|
+
|
|
230
|
+
```bash
|
|
231
|
+
errlore init claude-code # install Claude Code hooks + settings
|
|
232
|
+
errlore stats # memory stats for a data dir (--data-dir)
|
|
233
|
+
errlore lessons # list stored lessons
|
|
234
|
+
errlore --version
|
|
235
|
+
```
|
|
236
|
+
|
|
166
237
|
## API overview
|
|
167
238
|
|
|
168
239
|
The main entry point is `AgentMemory`. All other classes are internal --
|
|
@@ -176,7 +247,7 @@ you only need them for advanced use.
|
|
|
176
247
|
| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
|
|
177
248
|
| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized). |
|
|
178
249
|
| `lessons(limit)` | List all lessons (sorted by confidence). |
|
|
179
|
-
| `best_model(domain)` | Model with the highest trust weight
|
|
250
|
+
| `best_model(domain)` | Model with the highest trust weight *(experimental)*. |
|
|
180
251
|
| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`. |
|
|
181
252
|
| `pending_injections()` | Injections not yet reported. |
|
|
182
253
|
| `stats()` | Aggregate counts + trust weights. |
|
|
@@ -202,6 +273,49 @@ you only need them for advanced use.
|
|
|
202
273
|
(filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
|
|
203
274
|
(embedding metadata), `trust.json` (trust engine state).
|
|
204
275
|
|
|
276
|
+
## Security
|
|
277
|
+
|
|
278
|
+
A lesson is **trusted prompt content by design** — it is injected into your
|
|
279
|
+
prompts and reaches the model. So:
|
|
280
|
+
|
|
281
|
+
- **Do not ingest lessons from untrusted sources without review.** Treat lesson
|
|
282
|
+
capture like a code review, not like user input. A malicious lesson is a
|
|
283
|
+
prompt-injection vector — and this is the real control, not the sanitizer.
|
|
284
|
+
- **What the sanitizer does (and does not) do.** The lesson *pattern* passes
|
|
285
|
+
`sanitize_lesson_text`: it strips raw-JSON/code-fence *noise* and caps length
|
|
286
|
+
so log blobs don't pollute the prompt. It is a noise filter, **not** an
|
|
287
|
+
injection defense — it does not neutralize natural-language instructions, and
|
|
288
|
+
the *solution* text is stored as you author it (so it can hold real code).
|
|
289
|
+
Don't rely on it to make untrusted lessons safe.
|
|
290
|
+
- You control what becomes a lesson (`resolve(..., lesson=...)` /
|
|
291
|
+
`add_lesson(...)`); nothing is auto-promoted from raw model output.
|
|
292
|
+
|
|
293
|
+
Report security issues to the address in [SECURITY.md](SECURITY.md).
|
|
294
|
+
|
|
295
|
+
## Scale & limits (honest)
|
|
296
|
+
|
|
297
|
+
errlore is built for **one process, thousands of lessons** — a single agent or
|
|
298
|
+
a coding-agent session, not a high-throughput fleet. Know the edges:
|
|
299
|
+
|
|
300
|
+
- **`injections.jsonl` grows unbounded.** `report_outcome` scans the whole
|
|
301
|
+
ledger each call, so at very high injection volumes it slows down (roughly
|
|
302
|
+
linear in total injections). Fine for interactive/agent use; log compaction
|
|
303
|
+
is the next roadmap item. If you don't need the reinforcement loop, you can
|
|
304
|
+
ignore `report_outcome` and the file stays small.
|
|
305
|
+
- **Single-process by default.** The lesson/error stores use cross-process file
|
|
306
|
+
locks and are safe to share, but the **trust engine and the optional vector
|
|
307
|
+
index are not cross-process safe** — two processes writing `trust.json` /
|
|
308
|
+
`vectors.npy` concurrently can clobber each other (last-writer-wins). Run one
|
|
309
|
+
writer, or give each process its own `data_dir`. Multi-agent shared memory is
|
|
310
|
+
on the roadmap.
|
|
311
|
+
- **Embeddings index rebuild is O(n²) over many adds** — building a fresh index
|
|
312
|
+
over a large existing lesson store is slow the first time (then incremental).
|
|
313
|
+
- Concurrency is tested across threads; **multi-process** stress is not yet in
|
|
314
|
+
the suite.
|
|
315
|
+
|
|
316
|
+
None of these bite at the scale errlore targets today; they're stated so you
|
|
317
|
+
can decide, not discover.
|
|
318
|
+
|
|
205
319
|
## Roadmap
|
|
206
320
|
|
|
207
321
|
- [ ] Log compaction for injections journal
|