aegis-healer 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aegis_healer-0.1.0/.github/workflows/ci.yml +26 -0
- aegis_healer-0.1.0/.gitignore +17 -0
- aegis_healer-0.1.0/LICENSE +21 -0
- aegis_healer-0.1.0/PKG-INFO +375 -0
- aegis_healer-0.1.0/README.md +347 -0
- aegis_healer-0.1.0/docs/superpowers/plans/2026-11-19-aegis-healer.md +468 -0
- aegis_healer-0.1.0/docs/superpowers/plans/2026-11-20-hitl-pipeline.md +212 -0
- aegis_healer-0.1.0/docs/superpowers/plans/2026-11-20-meaning-drift.md +130 -0
- aegis_healer-0.1.0/examples/demo/README.md +40 -0
- aegis_healer-0.1.0/examples/demo/models.py +21 -0
- aegis_healer-0.1.0/examples/demo/released.jsonl +2 -0
- aegis_healer-0.1.0/examples/demo/v1.json +8 -0
- aegis_healer-0.1.0/examples/demo/v2.json +10 -0
- aegis_healer-0.1.0/examples/github-actions/README.md +95 -0
- aegis_healer-0.1.0/examples/github-actions/aegis-drift.yml +172 -0
- aegis_healer-0.1.0/examples/github-actions/list_pending.py +26 -0
- aegis_healer-0.1.0/examples/meaning/README.md +48 -0
- aegis_healer-0.1.0/examples/meaning/models.py +10 -0
- aegis_healer-0.1.0/examples/meaning/payments_v1.json +152 -0
- aegis_healer-0.1.0/examples/meaning/payments_v2.json +152 -0
- aegis_healer-0.1.0/examples/pipeline/README.md +63 -0
- aegis_healer-0.1.0/examples/pipeline/run_pipeline.py +51 -0
- aegis_healer-0.1.0/pyproject.toml +57 -0
- aegis_healer-0.1.0/src/aegis_healer/__init__.py +18 -0
- aegis_healer-0.1.0/src/aegis_healer/agent/__init__.py +81 -0
- aegis_healer-0.1.0/src/aegis_healer/agent/graph.py +137 -0
- aegis_healer-0.1.0/src/aegis_healer/agent/privacy.py +120 -0
- aegis_healer-0.1.0/src/aegis_healer/agent/prompts.py +88 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/__init__.py +44 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/check_cmd.py +171 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/common.py +175 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/init_cmd.py +63 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/profile_cmd.py +56 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/replay_cmd.py +113 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/review_cmd.py +184 -0
- aegis_healer-0.1.0/src/aegis_healer/cli/runs_cmd.py +38 -0
- aegis_healer-0.1.0/src/aegis_healer/codegen.py +175 -0
- aegis_healer-0.1.0/src/aegis_healer/detect.py +71 -0
- aegis_healer-0.1.0/src/aegis_healer/healer.py +797 -0
- aegis_healer-0.1.0/src/aegis_healer/mapping.py +149 -0
- aegis_healer-0.1.0/src/aegis_healer/matcher.py +84 -0
- aegis_healer-0.1.0/src/aegis_healer/meaning/__init__.py +1 -0
- aegis_healer-0.1.0/src/aegis_healer/meaning/config.py +47 -0
- aegis_healer-0.1.0/src/aegis_healer/meaning/detect.py +143 -0
- aegis_healer-0.1.0/src/aegis_healer/meaning/profile.py +187 -0
- aegis_healer-0.1.0/src/aegis_healer/meaning/resolve.py +25 -0
- aegis_healer-0.1.0/src/aegis_healer/notify.py +76 -0
- aegis_healer-0.1.0/src/aegis_healer/paths.py +118 -0
- aegis_healer-0.1.0/src/aegis_healer/pipeline.py +83 -0
- aegis_healer-0.1.0/src/aegis_healer/quarantine.py +302 -0
- aegis_healer-0.1.0/src/aegis_healer/runlog.py +81 -0
- aegis_healer-0.1.0/src/aegis_healer/store/__init__.py +89 -0
- aegis_healer-0.1.0/src/aegis_healer/store/files.py +238 -0
- aegis_healer-0.1.0/src/aegis_healer/types.py +117 -0
- aegis_healer-0.1.0/src/aegis_healer/validate.py +90 -0
- aegis_healer-0.1.0/tests/conftest.py +13 -0
- aegis_healer-0.1.0/tests/fakes.py +56 -0
- aegis_healer-0.1.0/tests/fixtures/__init__.py +0 -0
- aegis_healer-0.1.0/tests/fixtures/models.py +35 -0
- aegis_healer-0.1.0/tests/fixtures/payloads.py +32 -0
- aegis_healer-0.1.0/tests/test_agent.py +111 -0
- aegis_healer-0.1.0/tests/test_cli.py +292 -0
- aegis_healer-0.1.0/tests/test_codegen.py +138 -0
- aegis_healer-0.1.0/tests/test_detect.py +25 -0
- aegis_healer-0.1.0/tests/test_healer.py +266 -0
- aegis_healer-0.1.0/tests/test_list_pending.py +130 -0
- aegis_healer-0.1.0/tests/test_mapping.py +64 -0
- aegis_healer-0.1.0/tests/test_matcher.py +64 -0
- aegis_healer-0.1.0/tests/test_meaning_cli.py +281 -0
- aegis_healer-0.1.0/tests/test_meaning_config.py +62 -0
- aegis_healer-0.1.0/tests/test_meaning_detect.py +210 -0
- aegis_healer-0.1.0/tests/test_meaning_healer.py +519 -0
- aegis_healer-0.1.0/tests/test_meaning_profile.py +230 -0
- aegis_healer-0.1.0/tests/test_meaning_store.py +249 -0
- aegis_healer-0.1.0/tests/test_notify.py +234 -0
- aegis_healer-0.1.0/tests/test_paths.py +46 -0
- aegis_healer-0.1.0/tests/test_pipeline.py +176 -0
- aegis_healer-0.1.0/tests/test_privacy.py +246 -0
- aegis_healer-0.1.0/tests/test_quarantine.py +441 -0
- aegis_healer-0.1.0/tests/test_runlog.py +173 -0
- aegis_healer-0.1.0/tests/test_scale.py +110 -0
- aegis_healer-0.1.0/tests/test_store.py +84 -0
- aegis_healer-0.1.0/tests/test_usage.py +149 -0
- aegis_healer-0.1.0/tests/test_validate.py +38 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
os: [ubuntu-latest, windows-latest]
|
|
16
|
+
python: ["3.11", "3.12"]
|
|
17
|
+
runs-on: ${{ matrix.os }}
|
|
18
|
+
steps:
|
|
19
|
+
# Pin third-party actions to a commit SHA before making the repo public.
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python }}
|
|
24
|
+
- run: pip install -e ".[dev]"
|
|
25
|
+
- run: ruff check .
|
|
26
|
+
- run: pytest -q
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
.env
|
|
2
|
+
.venv/
|
|
3
|
+
venv/
|
|
4
|
+
__pycache__/
|
|
5
|
+
*.py[cod]
|
|
6
|
+
*.egg-info/
|
|
7
|
+
dist/
|
|
8
|
+
build/
|
|
9
|
+
.mypy_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.pytest_cache/
|
|
12
|
+
pyvenv.cfg
|
|
13
|
+
# Demo run state (commit .aegis/ in real projects)
|
|
14
|
+
examples/demo/.aegis/
|
|
15
|
+
examples/meaning/.aegis/
|
|
16
|
+
examples/meaning/released.jsonl
|
|
17
|
+
.superpowers/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Stuart Paul
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: aegis-healer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Aegis | API Healer: detect API schema drift and repair it with rules first, a LangGraph agent second.
|
|
5
|
+
Author: Stuart Paul
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: data engineering,data quality,langgraph,pydantic,schema drift
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Topic :: Database
|
|
11
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: httpx<1,>=0.27
|
|
14
|
+
Requires-Dist: langchain-anthropic<2,>=1.0
|
|
15
|
+
Requires-Dist: langchain-core<2,>=1.2.5
|
|
16
|
+
Requires-Dist: langgraph<2,>=1.0
|
|
17
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
18
|
+
Requires-Dist: python-dotenv<2,>=1.0
|
|
19
|
+
Requires-Dist: rich<16,>=13.7
|
|
20
|
+
Requires-Dist: typer<1,>=0.12.5
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: build; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
24
|
+
Requires-Dist: pyyaml; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
26
|
+
Requires-Dist: twine; extra == 'dev'
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# Aegis | API Healer
|
|
30
|
+
|
|
31
|
+
Detect when an API you depend on changes shape, and repair your pipeline's field mappings
|
|
32
|
+
automatically: plain rules first, a LangGraph agent (Claude Haiku) only for the hard cases,
|
|
33
|
+
and a validation gate in front of everything.
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install aegis-healer # once published; from a clone: pip install -e .
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Why
|
|
40
|
+
|
|
41
|
+
APIs drift: `user_name` becomes `userName`, an address moves under `contactInfo`, a date
|
|
42
|
+
becomes a Unix timestamp. Pipelines either crash or, worse, silently load bad data. Aegis
|
|
43
|
+
catches the drift, proposes a fix, proves the fix works on real records, and only then
|
|
44
|
+
applies it.
|
|
45
|
+
|
|
46
|
+
## Library
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import httpx
|
|
50
|
+
from aegis_healer import Healer
|
|
51
|
+
from app.models import User # your Pydantic model: the contract you expect
|
|
52
|
+
|
|
53
|
+
healer = Healer(auto_apply_threshold=0.9)
|
|
54
|
+
result = healer.validate(httpx.get(URL).json(), User, records_path="data")
|
|
55
|
+
|
|
56
|
+
result.status # "ok" | "healed" | "pending" | "failed"
|
|
57
|
+
result.records # list[User], validated
|
|
58
|
+
result.raise_for_status()
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## CLI
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
aegis init https://api.example.com/users --name User # generate models.py from a live response
|
|
65
|
+
aegis check https://api.example.com/users --model models:User [--threshold 0.9] [-H "Authorization: Bearer ..."] [--privacy masked] [--max-llm-calls 10] [--no-meaning]
|
|
66
|
+
aegis review [--approve ID] [--reject ID] [--keep ID] # interactive by default
|
|
67
|
+
aegis history models.User
|
|
68
|
+
aegis rollback models.User 2
|
|
69
|
+
aegis runs [--limit N] # run log
|
|
70
|
+
aegis replay --model models:User [--batch ID] [--out FILE] [--discard-dead]
|
|
71
|
+
aegis profile models.User # learned baseline per field
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
`aegis check` exits `0` (ok or healed), `1` (fix pending review) or `2` (failed), so it
|
|
75
|
+
drops straight into CI.
|
|
76
|
+
|
|
77
|
+
## How it works
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
detect → diff → rule matcher → (agent, only for what's left) → validate → apply or queue
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
1. **Detect**: validate records against your model and list missing, unexpected and retyped fields.
|
|
84
|
+
2. **Rule matcher**: fixes only drift with exactly one sensible answer: same name in a
|
|
85
|
+
different style (`userName` → `user_name`), same name in a new place
|
|
86
|
+
(`contact.email` → `email`), or a safe type conversion. If two fields could match, it
|
|
87
|
+
doesn't guess.
|
|
88
|
+
3. **Agent**: a LangGraph loop (propose → check → retry with the errors) on the fields the
|
|
89
|
+
rules couldn't settle. It can also declare a field **unmappable** with a reason, e.g.
|
|
90
|
+
a UUID string can't become an `int` without inventing data.
|
|
91
|
+
4. **Validate**: the full fix is applied to real sample records and must pass your model.
|
|
92
|
+
5. **Apply or queue**: rule fixes need no approval. Agent fixes apply automatically only if
|
|
93
|
+
every one is at or above `auto_apply_threshold` (default `0.9`; `1.0` means a human
|
|
94
|
+
approves every agent fix). Everything else waits in `aegis review`.
|
|
95
|
+
|
|
96
|
+
## Self-healing pipeline
|
|
97
|
+
|
|
98
|
+
`Pipeline` wraps fetch, heal and load, and holds back records you can't load safely yet:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from aegis_healer import Healer, Pipeline
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def fetch():
|
|
105
|
+
return httpx.get(URL).json()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def sink(records): # must be idempotent: upsert by key
|
|
109
|
+
...
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
run = Pipeline(fetch, User, sink, healer=Healer(), records_path="data").run()
|
|
113
|
+
run.status, run.loaded, run.quarantined, run.pending_id
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
drift -> rules -> agent -> confidence >= threshold ?
|
|
118
|
+
yes: apply -> sink
|
|
119
|
+
no: pending fix + quarantine records + notify
|
|
120
|
+
-> human review (approve) -> replay quarantined records -> sink
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Each run fetches, heals, loads valid records, then replays quarantined records whose fix was
|
|
124
|
+
approved since. Sinks must be idempotent because an API may re-serve records. While a fix is
|
|
125
|
+
pending, later runs reuse it (no new agent calls, no second pending fix). See
|
|
126
|
+
`examples/pipeline`.
|
|
127
|
+
|
|
128
|
+
Use one writer per `.aegis/` folder. If you also run the GitHub PR flow
|
|
129
|
+
(`examples/github-actions`) against the same `.aegis/`, pull the merged approvals into the
|
|
130
|
+
Pipeline host before its next run, and don't let both write mappings at the same time.
|
|
131
|
+
|
|
132
|
+
## Human review
|
|
133
|
+
|
|
134
|
+
- **CLI**: `aegis review` (interactive), or `aegis review --approve ID` / `--reject ID`
|
|
135
|
+
(and `--keep ID` for meaning fixes, see Meaning drift).
|
|
136
|
+
- **Webhook**: get pinged when a fix goes pending (see Notifications). `AEGIS_WEBHOOK_URL`
|
|
137
|
+
is read by the `aegis` CLI only; library and `Pipeline` users pass
|
|
138
|
+
`Healer(on_pending=webhook_notifier(url))`.
|
|
139
|
+
- **Pull requests**: `examples/github-actions` runs `aegis check` on a schedule and opens a
|
|
140
|
+
PR per fix. Merging the PR is the approval; closing it means "not this fix" (the next
|
|
141
|
+
run proposes again unless you change the model or set `AEGIS_PAUSED=true`).
|
|
142
|
+
|
|
143
|
+
## Meaning drift
|
|
144
|
+
|
|
145
|
+
Same field name, same type, different meaning: `amount` goes from dollars to cents, or
|
|
146
|
+
timestamps go from UTC to local time. Schema validation passes, so this is the quiet kind of
|
|
147
|
+
break. Aegis learns a baseline per field from healthy runs (aggregate counts only: size
|
|
148
|
+
buckets, decimal places, timezone classes, short lowercase or uppercase category words; never
|
|
149
|
+
raw numbers, text, IDs or dates; stored in `.aegis/profiles/`, see `aegis profile MODEL_KEY`)
|
|
150
|
+
and compares each new batch to it. No LLM is involved.
|
|
151
|
+
|
|
152
|
+
What can end up in committed files: category words are any short all-lowercase or
|
|
153
|
+
all-uppercase word (letters, `_`, `-`, at most 20 characters), so besides status codes like
|
|
154
|
+
`active` or `USD`, a field of short usernames or codes could store some of them in
|
|
155
|
+
`.aegis/profiles/` and in the findings of `.aegis/pending/` files. Set such fields to `off`
|
|
156
|
+
(below) if that matters. Numeric strings such as `"12.50"` are treated as text: they are not
|
|
157
|
+
scale-checked.
|
|
158
|
+
|
|
159
|
+
It catches:
|
|
160
|
+
|
|
161
|
+
- **Scale**: a number field about 100x or 1000x bigger or smaller (dollars/cents,
|
|
162
|
+
seconds/milliseconds, 0.05/5 percent). Suggested fix: a `scale` on the mapping.
|
|
163
|
+
- **Timezone**: a datetime field whose dominant timezone class changed (UTC to offset or
|
|
164
|
+
naive, or any other switch).
|
|
165
|
+
- **Category**: a category field that suddenly uses new values (`active` becomes `A`).
|
|
166
|
+
|
|
167
|
+
A field needs 20 records of history before it is checked, and a batch needs 5 records.
|
|
168
|
+
Every run is checked, including runs whose schema fix is pending or failed: valid records
|
|
169
|
+
with meaning drift are never loaded. The baseline only learns from fully healthy runs (no
|
|
170
|
+
findings, status `ok` or `healed`), so it can't learn the drift. The affected records are
|
|
171
|
+
blocked and quarantined in their own batch, and a pending meaning fix is created. On a
|
|
172
|
+
schema-clean run the status becomes `pending` (exit code 1); on a run whose schema fix is
|
|
173
|
+
pending or failed the status (and exit code) stays, `HealResult.meaning_pending_id` names the
|
|
174
|
+
meaning fix and the message ends with `; meaning drift needs review: <fields>`. Meaning fixes
|
|
175
|
+
are never auto-applied, whatever the threshold. If the API drifts in schema again before you
|
|
176
|
+
decide, the new meaning fix for the same drift replaces the old one (marked `superseded`) and
|
|
177
|
+
takes over its quarantined records, so one decision releases them all.
|
|
178
|
+
|
|
179
|
+
Choose what happens per field in `.aegis/config.toml` (`block` is the default):
|
|
180
|
+
|
|
181
|
+
```toml
|
|
182
|
+
[meaning]
|
|
183
|
+
default = "block" # block | flag | off
|
|
184
|
+
|
|
185
|
+
[meaning.fields]
|
|
186
|
+
amount = "block" # a bare field path
|
|
187
|
+
"app.models.Payment.note" = "flag" # model key + path wins over a bare path
|
|
188
|
+
created_at = "off"
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
`block` quarantines the records, `flag` loads them but still opens the pending fix and prints
|
|
192
|
+
the finding, `off` skips the field. `aegis check --no-meaning` skips all meaning checks for
|
|
193
|
+
one run.
|
|
194
|
+
|
|
195
|
+
Resolve a meaning fix in `aegis review`:
|
|
196
|
+
|
|
197
|
+
- **[a]pply fix**: the suggested scale becomes a mapping (it composes with renames), then
|
|
198
|
+
`aegis replay` releases the blocked records converted. Only offered when there is a
|
|
199
|
+
suggested fix.
|
|
200
|
+
- **[k]eep data** (or `aegis review --keep ID`): the new values are the new normal. The
|
|
201
|
+
baseline for the field is reset and rebuilds from the next runs; the blocked records can
|
|
202
|
+
then be replayed as they are. Use this when the API really changed and your model expects it.
|
|
203
|
+
- **[r]eject**: the blocked records are never released; their batch becomes dead. Delete it
|
|
204
|
+
with `aegis replay --model ... --discard-dead`.
|
|
205
|
+
- **[s]kip**: decide later. Blocked records stay quarantined.
|
|
206
|
+
|
|
207
|
+
Known limitations:
|
|
208
|
+
|
|
209
|
+
- Timezone and category drift have no automatic fix: keep the data, or fix it upstream.
|
|
210
|
+
- A field needs 20 records of history before it is checked.
|
|
211
|
+
- Records are meaning-checked when a run first sees them. Only records quarantined for
|
|
212
|
+
*schema* drift and later released by replay (after the schema fix is approved) skip the
|
|
213
|
+
check: replay re-validates them against the mappings, not against the baseline.
|
|
214
|
+
- Small batches (5 records) against a very wide baseline can occasionally false-alarm; set
|
|
215
|
+
noisy fields to `flag` or `off`.
|
|
216
|
+
|
|
217
|
+
See `examples/meaning` for a runnable dollars-to-cents demo.
|
|
218
|
+
|
|
219
|
+
## Privacy
|
|
220
|
+
|
|
221
|
+
What the agent sees of your API data is set by `--privacy` / `Healer(privacy=)`:
|
|
222
|
+
|
|
223
|
+
| Mode | Sent to the LLM |
|
|
224
|
+
|---|---|
|
|
225
|
+
| `full` | Real sample values |
|
|
226
|
+
| `masked` (default) | Values replaced by shape-preserving placeholders |
|
|
227
|
+
| `schema` | Field names and types only, no values |
|
|
228
|
+
|
|
229
|
+
Residual risk in `masked`: short single-word values such as `"active"` and epoch-like
|
|
230
|
+
numbers pass through by design, because the agent needs them to spot status flags and
|
|
231
|
+
timestamps. For highly sensitive APIs use `--privacy schema`.
|
|
232
|
+
|
|
233
|
+
## Budget and cost
|
|
234
|
+
|
|
235
|
+
`--max-llm-calls` (default 10) / `Healer(max_llm_calls=)` caps agent calls per run.
|
|
236
|
+
`HealResult.usage` (calls, tokens, est_cost_usd) is shown in the `aegis check` output.
|
|
237
|
+
At Haiku 5.5's $0.10 / $0.50 per million input / output tokens, an agent call costs about
|
|
238
|
+
$0.0005. The agent only runs when the rules can't fix the drift.
|
|
239
|
+
|
|
240
|
+
## Run log
|
|
241
|
+
|
|
242
|
+
Every check appends a line to `.aegis/runs.jsonl`; `aegis runs` shows recent runs.
|
|
243
|
+
Messages are categorical (no API values) unless you use `--privacy full`.
|
|
244
|
+
|
|
245
|
+
## Quarantine & replay
|
|
246
|
+
|
|
247
|
+
Records that can't be loaded yet are saved under `.aegis/quarantine/` (gitignored: it may
|
|
248
|
+
hold PII), in two kinds of batch:
|
|
249
|
+
|
|
250
|
+
- **Schema** batches hold records that failed validation. Replay releases every record that
|
|
251
|
+
validates with the current mappings, whatever happened to the fix (approved, rejected,
|
|
252
|
+
or a later fix that heals them).
|
|
253
|
+
- **Meaning** batches hold records that validated but whose values changed meaning. They
|
|
254
|
+
wait on their meaning fix: released only once it is approved or kept; still blocked while
|
|
255
|
+
it is pending (or missing). A batch of a rejected meaning fix is **dead**: replay skips it
|
|
256
|
+
and doesn't count it as blocked; delete it with `aegis replay --discard-dead`.
|
|
257
|
+
|
|
258
|
+
`Pipeline.run()` replays them into your sink and changes the quarantine only after the
|
|
259
|
+
insert succeeded. Release is partial: every record that now validates is released; a batch
|
|
260
|
+
with records that still fail (say, one value no mapping can fix) keeps just those records,
|
|
261
|
+
under the same batch id.
|
|
262
|
+
|
|
263
|
+
`aegis replay --model models:User` previews what would be released without changing
|
|
264
|
+
anything. With `--out FILE` (a new file; existing files are never overwritten) it writes the
|
|
265
|
+
released records, de-duplicated, **then removes cleared batches and shrinks partial ones**.
|
|
266
|
+
|
|
267
|
+
Quarantined records are kept until they are replayed: there is no automatic purge yet.
|
|
268
|
+
`aegis replay --model ... --discard-dead` deletes dead meaning batches; to discard everything,
|
|
269
|
+
delete `.aegis/quarantine/`.
|
|
270
|
+
|
|
271
|
+
## Notifications
|
|
272
|
+
|
|
273
|
+
For the `aegis` CLI, set `AEGIS_WEBHOOK_URL` (https; http only for localhost). In Python
|
|
274
|
+
(`Healer`, `Pipeline`) the variable is not read: pass
|
|
275
|
+
`Healer(on_pending=webhook_notifier(url))` (from `aegis_healer.notify`). The POST body is
|
|
276
|
+
JSON with a `text` field, so Slack and Discord-style webhooks work. Redirects are never
|
|
277
|
+
followed, and proxy and custom CA environment variables are ignored on purpose, so a
|
|
278
|
+
notification can't be rerouted.
|
|
279
|
+
|
|
280
|
+
## Get an Anthropic API key
|
|
281
|
+
|
|
282
|
+
1. Open the Console at platform.claude.com and sign in.
|
|
283
|
+
2. Settings, Billing: add credit.
|
|
284
|
+
3. Workspaces: create a workspace named `aegis`, and set a Monthly limit.
|
|
285
|
+
4. API Keys: Create key in that workspace. Copy it now; it is shown once and starts `sk-ant-`.
|
|
286
|
+
5. Put it in `.env` (gitignored): `ANTHROPIC_API_KEY=sk-ant-...`
|
|
287
|
+
|
|
288
|
+
## Writing your own Proposer
|
|
289
|
+
|
|
290
|
+
Any object with `propose(request, check, budget=None)` and a `last_usage` (`Usage`) works
|
|
291
|
+
(an older `propose(request, check)` is still called, without the budget argument):
|
|
292
|
+
return a patch that passed `check(patch)`, or raise `AgentError`. Honour `budget` (max LLM
|
|
293
|
+
calls for this invocation) and set `last_usage` for each call, even when it raised, so
|
|
294
|
+
the Healer can enforce the run budget. Pass it as `Healer(proposer=...)`.
|
|
295
|
+
|
|
296
|
+
## Where fixes live
|
|
297
|
+
|
|
298
|
+
```
|
|
299
|
+
.aegis/
|
|
300
|
+
mappings/<model>.json current mappings
|
|
301
|
+
history/<model>/0001.json every version, never edited
|
|
302
|
+
pending/0001.json fixes waiting for review
|
|
303
|
+
quarantine/ held-back records (gitignored)
|
|
304
|
+
runs.jsonl run log
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
Plain JSON, written atomically. Commit `.aegis/` to git and every mapping change becomes
|
|
308
|
+
reviewable, diffable and revertible. In read-only deployments use
|
|
309
|
+
`Healer(persist=False)` (or `aegis check --dry-run`) to heal in memory without writing.
|
|
310
|
+
The store sits behind an interface, so a shared database backend can be added for teams
|
|
311
|
+
running many workers.
|
|
312
|
+
|
|
313
|
+
## Security model
|
|
314
|
+
|
|
315
|
+
- **Fixes are data, not code.** A fix is a list of mappings with a transform from a fixed
|
|
316
|
+
allow-list: `none, to_int, to_float, to_str, to_bool, parse_datetime, epoch_to_iso,
|
|
317
|
+
value_map, concat`. LLM output is never executed.
|
|
318
|
+
- **API data is untrusted.** Payload samples are truncated and wrapped as data in the
|
|
319
|
+
prompt. The agent may only map fields it was asked about, from fields that exist, and
|
|
320
|
+
every fix must pass validation, so a payload that says "ignore your instructions" can't
|
|
321
|
+
produce an applied change. Keys are JSON-quoted and `<`/`>` escaped so payload text
|
|
322
|
+
can't forge prompt sections. A proposal that fails its checks is never returned, and
|
|
323
|
+
the Healer re-checks scope before applying anything.
|
|
324
|
+
- **Secrets**: `ANTHROPIC_API_KEY` is read from the environment only. CLI header values
|
|
325
|
+
are never printed.
|
|
326
|
+
- **Fetching**: `http`/`https` only, 30 s overall deadline, 10 MB cap, at most 5 redirects.
|
|
327
|
+
Redirects are followed by hand: your `--header` values are only sent to the original
|
|
328
|
+
host, and https never downgrades to http. Query strings are hidden in output.
|
|
329
|
+
- **Terminal output**: text that came from the API or the LLM is escaped before printing,
|
|
330
|
+
so it can't fake rows or hide text in `aegis review`.
|
|
331
|
+
- **`aegis init`** turns untrusted API keys into Python: field names are sanitised to
|
|
332
|
+
identifiers, and original keys only appear as string aliases.
|
|
333
|
+
|
|
334
|
+
## Configuration
|
|
335
|
+
|
|
336
|
+
| Setting | Where |
|
|
337
|
+
|---|---|
|
|
338
|
+
| Auto-apply threshold | `Healer(auto_apply_threshold=)` / `--threshold` |
|
|
339
|
+
| Anthropic key | `ANTHROPIC_API_KEY` (the CLI also reads `.env`) |
|
|
340
|
+
| Model | `AEGIS_MODEL` (default `claude-haiku-5-5`) |
|
|
341
|
+
| Store folder | `AEGIS_DIR` / `--db` (default `.aegis`) |
|
|
342
|
+
|
|
343
|
+
Without an API key, Aegis runs in rules-only mode.
|
|
344
|
+
|
|
345
|
+
## Limitations (v0.1)
|
|
346
|
+
|
|
347
|
+
- Drift *inside* lists of objects (`items[].qty`) is detected as failed records but not
|
|
348
|
+
repaired; lists are treated as single values.
|
|
349
|
+
- Models with `extra="forbid"` fail when the API adds a field; there's no "drop" mapping yet.
|
|
350
|
+
- Meaning drift detection is statistical, see its limits under Meaning drift.
|
|
351
|
+
|
|
352
|
+
## Development
|
|
353
|
+
|
|
354
|
+
Windows PowerShell:
|
|
355
|
+
|
|
356
|
+
```powershell
|
|
357
|
+
python -m venv .venv
|
|
358
|
+
.venv\Scripts\Activate.ps1
|
|
359
|
+
pip install -e ".[dev]"
|
|
360
|
+
pytest
|
|
361
|
+
ruff check .
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
macOS/Linux: `source .venv/bin/activate` instead of the second line.
|
|
365
|
+
|
|
366
|
+
See `examples/demo` for an end-to-end run against the original Phase 1 mock API data.
|
|
367
|
+
The Phase 1 Docker/FastAPI/Postgres build is kept for reference in `legacy/phase1/`.
|
|
368
|
+
|
|
369
|
+
## Roadmap
|
|
370
|
+
|
|
371
|
+
- Shared store backends (Postgres) and run-result sinks (Databricks Delta, S3 + Athena)
|
|
372
|
+
|
|
373
|
+
## License
|
|
374
|
+
|
|
375
|
+
MIT
|