agent-audit-memory 0.2.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_audit_memory-0.2.0rc1.dist-info/METADATA +236 -0
- agent_audit_memory-0.2.0rc1.dist-info/RECORD +16 -0
- agent_audit_memory-0.2.0rc1.dist-info/WHEEL +5 -0
- agent_audit_memory-0.2.0rc1.dist-info/licenses/LICENSE +21 -0
- agent_audit_memory-0.2.0rc1.dist-info/top_level.txt +1 -0
- aml/__init__.py +56 -0
- aml/chain.py +99 -0
- aml/hardrules.py +48 -0
- aml/ledger.py +82 -0
- aml/lock.py +48 -0
- aml/recycle.py +121 -0
- aml/search.py +45 -0
- aml/security.py +97 -0
- aml/store.py +166 -0
- aml/util.py +156 -0
- aml/wal.py +94 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agent-audit-memory
|
|
3
|
+
Version: 0.2.0rc1
|
|
4
|
+
Summary: A verifiable memory layer for AI agents: WAL with fsync durability, hash-chained trail, recycle bin + provable erasure, conflict ledger, write lock, hard rules, single-writer process lock (research prototype / AI-assisted development).
|
|
5
|
+
Author-email: agent-memory-ledger maintainers <2307788504@qq.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://gitcode.com/gcw_MKhSyoGN/agent-memory-ledger
|
|
8
|
+
Keywords: agent,memory,mcp,audit,verifiable,llm,wal
|
|
9
|
+
Requires-Python: >=3.9
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Dynamic: license-file
|
|
13
|
+
|
|
14
|
+
# agent-memory-ledger
|
|
15
|
+
|
|
16
|
+
> **A verifiable memory layer for AI agents** — WAL-first writes, hash-chained
|
|
17
|
+
> trail, soft-delete recycle bin, conflict ledger, write locks and permanent
|
|
18
|
+
> rigid rules, with a **fail-closed** posture.
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install agent-audit-memory # zero deps, Python 3.9+
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
> **PyPI name note**: on PyPI this package is published as
|
|
25
|
+
> `agent-audit-memory` — the earlier `agent-memory-ledger` name was already
|
|
26
|
+
> taken on PyPI by an unrelated project, and the rename also follows the
|
|
27
|
+
> naming review recommendation: `audit` communicates the actual value
|
|
28
|
+
> (auditable memory) and avoids the "ledger/blockchain" association. The repo,
|
|
29
|
+
> import path (`import aml`), CLI and everything else are unchanged.
|
|
30
|
+
|
|
31
|
+
> **Release policy**: until a first external-user validation window closes, PyPI
|
|
32
|
+
> releases are published as **pre-release** (`==0.2.0rc1` etc.), not `latest`.
|
|
33
|
+
> Pre-release builds are suitable for prototype validation, **not** for
|
|
34
|
+
> processing sensitive data. See `docs/performance.md` and
|
|
35
|
+
> `docs/compliance.md` before choosing durability/backends.
|
|
36
|
+
|
|
37
|
+

|
|
38
|
+

|
|
39
|
+

|
|
40
|
+

|
|
41
|
+

|
|
42
|
+
|
|
43
|
+
<a href="https://gitcode.com/gcw_MKhSyoGN/agent-memory-ledger"><img src="https://img.shields.io/badge/GitCode-gcw_MKhSyoGN%2Fagent-memory-ledger-blue" alt="GitCode"></a>
|
|
44
|
+
<a href="https://github.com/langarden/agent-memory-ledger"><img src="https://img.shields.io/badge/GitHub-langarden%2Fagent-memory-ledger-black" alt="GitHub"></a>
|
|
45
|
+
|
|
46
|
+
[中文版](docs/README.zh-CN.md) · *Research prototype / AI-assisted development.*
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
Most agent memory stores help you **remember**. This one also helps you
|
|
51
|
+
**prove** that nothing was recorded wrong, deleted silently, or tampered with —
|
|
52
|
+
an **audit-grade memory** for agents that need to be accountable.
|
|
53
|
+
|
|
54
|
+
## Known limitations (read first)
|
|
55
|
+
|
|
56
|
+
> ⚠️ v0.2 is a research prototype. Read this before filing an issue.
|
|
57
|
+
|
|
58
|
+
| # | Limitation | Effect / guidance |
|
|
59
|
+
|:--|:--|:--|
|
|
60
|
+
| 1 | **Single-process writer** per storage directory | enforced by an exclusive lock; second writer raises `ProcessContentionError` with guidance. Recommended: one process, one directory (or per-process `namespace=`). |
|
|
61
|
+
| 2 | **No NFS / network shares** as storage backend | atomic rename is unreliable there; local disk only. |
|
|
62
|
+
| 3 | **File-store capacity** | tuned for ≤ ~10k records per JSON file; SQLite backend on roadmap. |
|
|
63
|
+
| 4 | **No read-side access control** | `namespace=` is data isolation, NOT a security boundary; add OS/object-store ACLs for multi-tenant production. |
|
|
64
|
+
| 5 | **Not a legal forensics tool** | erasure certificates / hash chains are internal integrity proofs, not court-grade evidence (see `docs/compliance.md`). |
|
|
65
|
+
| 6 | **`durability="strict"` is slow** | fsync per write; use it for audit scenarios, not high-QPS chat agents. See `docs/performance.md`. |
|
|
66
|
+
| 7 | **Chain `verify()` is O(n)** | fine ≤ ~100k entries; segmented/snapshot verifier on roadmap. |
|
|
67
|
+
|
|
68
|
+
## Why another memory library?
|
|
69
|
+
|
|
70
|
+
| Agent memory stores do… | agent-memory-ledger also does… |
|
|
71
|
+
|:--|:--|
|
|
72
|
+
| store and retrieve memories | **write-ahead log** — crash replayable, nothing lost |
|
|
73
|
+
| remember long conversations | **hash-chained trail** — offline `verify()` catches tampering |
|
|
74
|
+
| delete old entries | **soft delete + recycle bin** — 30-day retention, restorable |
|
|
75
|
+
| update conflicting values | **conflict ledger** — surfaces conflicts for human resolution, never merges silently |
|
|
76
|
+
| let agents write freely | **write locks** — subagents write `draft`, master promotes to `core` |
|
|
77
|
+
| decay old knowledge | **hard rules** — permanent rigid rules never decay, archive, or filter |
|
|
78
|
+
|
|
79
|
+
### The gap this fills
|
|
80
|
+
|
|
81
|
+
Reliability failures are not edge cases. Common failure modes across popular
|
|
82
|
+
agent-memory stores include:
|
|
83
|
+
|
|
84
|
+
- **A failed write loses the memory body permanently** and leaves no way to
|
|
85
|
+
retry — exactly what a write-ahead log + crash replay prevents.
|
|
86
|
+
- **Concurrent writers tear files / corrupt state** during shutdown or shared
|
|
87
|
+
use — a single append-only WAL with an exclusive writer lock removes this
|
|
88
|
+
class of failure.
|
|
89
|
+
- **A single corrupted character silently invalidates a whole batch** —
|
|
90
|
+
silent-corruption failures that a hash-chained, offline-verifiable trail
|
|
91
|
+
surfaces immediately instead of propagating.
|
|
92
|
+
|
|
93
|
+
These are **reliability and verifiability** problems, not "remember harder"
|
|
94
|
+
problems. `agent-memory-ledger` is built so that **every write is logged before
|
|
95
|
+
it lands, every record is hash-chained and offline-checkable, nothing is
|
|
96
|
+
deleted silently, and no conflict is merged without a human** — the ten
|
|
97
|
+
invariants below.
|
|
98
|
+
|
|
99
|
+
## Product family
|
|
100
|
+
|
|
101
|
+
Part of the **Verifiable-agent family** — the same fail-closed, audit-trail
|
|
102
|
+
design language across four repositories (READMEs link each other):
|
|
103
|
+
|
|
104
|
+
| Repo | Role |
|
|
105
|
+
|:--|:--|
|
|
106
|
+
| [agent-call-audit](https://gitcode.com/gcw_MKhSyoGN/agent-call-audit) | audit **engine** for agent tool calls |
|
|
107
|
+
| **agent-memory-ledger** | verifiable **memory** layer |
|
|
108
|
+
| [agent-audit-gateway](https://gitcode.com/gcw_MKhSyoGN/agent-audit-gateway) | HTTP **gateway** around the engine |
|
|
109
|
+
| [agent-route-grid](https://gitcode.com/gcw_MKhSyoGN/agent-route-grid) | audited multi-agent **routing** |
|
|
110
|
+
|
|
111
|
+
## Install
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
pip install agent-audit-memory # once published
|
|
115
|
+
# dev / from source:
|
|
116
|
+
pip install -e .
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Zero runtime dependencies. Python 3.9+.
|
|
120
|
+
|
|
121
|
+
## 5-line example
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from aml import MemoryStore, RecycleBin
|
|
125
|
+
|
|
126
|
+
store = MemoryStore("./mem") # WAL + hash chain on disk
|
|
127
|
+
store.put({"layer": "core", "key": "redline", "payload": "no leverage"})
|
|
128
|
+
print(store.chain.verify()) # (True, None, 1) offline proof
|
|
129
|
+
bin = RecycleBin(store); bin.recycle("core", "redline"); bin.restore("redline")
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Proof of life
|
|
133
|
+
|
|
134
|
+
Real output from `python examples/demo.py` (seven scenarios):
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
01 write + verify trail -> trail verify: (True, None, 2)
|
|
138
|
+
02 WAL replay after crash -> replayed: 2 bad: 0 applied_keys: ['risk_redline', ...]
|
|
139
|
+
03 hard rule exemption -> risk_redline score=1.0 (kept) | note/alpha score=0.5
|
|
140
|
+
04 conflict resolution -> conflict CFL-... pending -> after resolve pending: []
|
|
141
|
+
05 recycle + restore -> original layer has key: False -> restored: True
|
|
142
|
+
06 write lock fail-closed -> subagent write core: False -> denied (fail-closed)
|
|
143
|
+
07 sanitize + audit -> {'password': '[REDACTED]', 'token': '[REDACTED]'}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## Invariants
|
|
147
|
+
|
|
148
|
+
| # | Invariant | Guarantee |
|
|
149
|
+
|:--|:--|:--|
|
|
150
|
+
| 1 | WAL before commit | every mutation is logged first; crash replayable |
|
|
151
|
+
| 2 | Triple pre-check | conflict / sensitive / redundant checks gate every write |
|
|
152
|
+
| 3 | Soft delete | records go to a recycle bin (30 days), restorable |
|
|
153
|
+
| 4 | Hash-chained trail | `verify()` recomputes the chain; tamper is detected |
|
|
154
|
+
| 5 | Hard rules | permanent rigid rules never decay, archive, or filter |
|
|
155
|
+
| 6 | Write lock | subagents write `draft` only; master promotes to `core` |
|
|
156
|
+
| 7 | Zero sensitive persistence | secrets are sanitized / stored as hashes only |
|
|
157
|
+
| 8 | Conflicts surfaced | pending conflicts require human resolution; never merged silently |
|
|
158
|
+
| 9 | Single-writer enforcement | exclusive cross-process lock per storage dir (fail-closed) |
|
|
159
|
+
| 10 | Provable erasure | purge writes an erasure certificate (digest only), plaintext removed |
|
|
160
|
+
|
|
161
|
+
**Durability** (`durability=` on `MemoryStore` / `WalLog`): `strict` fsyncs
|
|
162
|
+
every append (power-loss safe, audit-grade), `normal` flushes per append
|
|
163
|
+
(default), `fast` skips fsync (testing only).
|
|
164
|
+
|
|
165
|
+
## Concurrency & capacity limits (current file backend)
|
|
166
|
+
|
|
167
|
+
- **Single-process writer** per storage directory (enforced by an exclusive
|
|
168
|
+
lock — a second writer raises `ProcessContentionError` instead of corrupting
|
|
169
|
+
files). This is the **recommended** v0.2 posture: one agent process, one
|
|
170
|
+
directory. A shared SQLite backend for multi-process write is on the roadmap.
|
|
171
|
+
- JSON file store is tuned for **≤ ~10k records per file**; above that, switch
|
|
172
|
+
to the planned SQLite backend (WAL mode, transactional, indexed).
|
|
173
|
+
- Chain `verify()` is an O(n) linear pass — fine for ≤ ~100k entries; a
|
|
174
|
+
segmented/snapshot verifier is on the roadmap for larger sets.
|
|
175
|
+
|
|
176
|
+
## Compliance note (read before citing regulations)
|
|
177
|
+
|
|
178
|
+
This library provides the **technical substrate** for audit-grade memory —
|
|
179
|
+
WAL, hash-chained trail, soft delete + provable erasure, write locks. It does
|
|
180
|
+
**not** by itself make a system "GDPR / EU AI Act compliant": full compliance
|
|
181
|
+
also requires retention policies, access control on read, operator
|
|
182
|
+
procedures, and organisational processes. See `docs/compliance.md` and
|
|
183
|
+
`RETAIN_ERASE_POLICY.md` for the current capability envelope. (Managed by
|
|
184
|
+
expert review 2026-11.)
|
|
185
|
+
|
|
186
|
+
## Architecture
|
|
187
|
+
|
|
188
|
+
```
|
|
189
|
+
aml/
|
|
190
|
+
├── wal.py append-only write-ahead log + crash recovery
|
|
191
|
+
├── chain.py hash-chained audit trail (LSN-ordered), offline-verifiable
|
|
192
|
+
├── store.py layered store (draft / hot / core / archive; namespace-isolated)
|
|
193
|
+
├── recycle.py soft delete + 30-day retention + provable erasure certificates
|
|
194
|
+
├── ledger.py conflict ledger (pending -> human resolution)
|
|
195
|
+
├── lock.py role-based write permission (fail-closed)
|
|
196
|
+
├── hardrules.py permanent rigid rules (exempt from decay/archive)
|
|
197
|
+
├── search.py layered retrieval (hard first + source receipt)
|
|
198
|
+
├── security.py digest / sanitize / backup / audit events
|
|
199
|
+
└── util.py atomic writes + exclusive cross-process file lock
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
## Roadmap ideas
|
|
203
|
+
|
|
204
|
+
- **v0.3 — audit-grade completeness** (per external expert review 2026-11):
|
|
205
|
+
content-level read access control; provenance chain (which prompt / tool
|
|
206
|
+
call produced a memory); configurable retention tiers + auto-archive; export
|
|
207
|
+
of an independent audit-event stream.
|
|
208
|
+
- **Scalability — SQLite backend (architecture constraint)**: when adding the
|
|
209
|
+
SQLite store, **do not re-implement the JSON-file hash-chain at file level**.
|
|
210
|
+
SQLite already provides WAL + transactions + row locks; the hash-chain
|
|
211
|
+
semantic should be pushed down to **row level** rather than stacking a second
|
|
212
|
+
WAL on top. Two nested WAL layers add avoidable complexity and consistency
|
|
213
|
+
risk.
|
|
214
|
+
- **Ecosystem — LangGraph/LangChain `Checkpointer` binding (contract
|
|
215
|
+
boundary)**: the compatibility layer stores LangGraph checkpoints **as aml
|
|
216
|
+
payloads** and aml owns the integrity/audit of those writes. It does **not**
|
|
217
|
+
change LangGraph checkpoint semantics; LangGraph-internal bugs / checkpoint
|
|
218
|
+
overwrites are outside aml's guarantees. A **separate demo repository**
|
|
219
|
+
(not the main repo) will host the LangGraph integration demo + compliance
|
|
220
|
+
export scripts, keeping the main repo clean so users can clone-and-run.
|
|
221
|
+
- **MCP server** (`memory.read` / `memory.write`); **remote sync** with a
|
|
222
|
+
verifiable append-only chain.
|
|
223
|
+
|
|
224
|
+
## Contributing / Security / License / AI disclosure
|
|
225
|
+
|
|
226
|
+
- [CONTRIBUTING.md](CONTRIBUTING.md) — fail-closed first, no new deps,
|
|
227
|
+
sanitized commits, green tests.
|
|
228
|
+
- [SECURITY.md](SECURITY.md) — report privately; reference implementation,
|
|
229
|
+
not a hardened product; no third-party security audit yet.
|
|
230
|
+
- [docs/compliance.md](docs/compliance.md) — capability envelope vs EU AI Act
|
|
231
|
+
/ GDPR; [RETAIN_ERASE_POLICY.md](RETAIN_ERASE_POLICY.md) — retention &
|
|
232
|
+
provable-erasure policy; [docs/AUDIT_LOG_FORMAT.md](docs/AUDIT_LOG_FORMAT.md)
|
|
233
|
+
— log schema for third-party audit tooling.
|
|
234
|
+
- [docs/AIGC_DISCLOSURE.md](docs/AIGC_DISCLOSURE.md) — AI-assisted development
|
|
235
|
+
disclosure.
|
|
236
|
+
- [LICENSE](LICENSE) — MIT, `agent-memory-ledger maintainers`.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
agent_audit_memory-0.2.0rc1.dist-info/licenses/LICENSE,sha256=znEIljhDhuHckQVTr_re7-gsW_Mhka4NJct0IKH9zpA,1087
|
|
2
|
+
aml/__init__.py,sha256=mfaPtZjOnDhcZEBZMJgt_AdQGvCELx0AzqMTi95qS_M,1917
|
|
3
|
+
aml/chain.py,sha256=YEsmOy6pLroGOQ0KmauyxTO9l2i-UDrZ7lIOlM-454U,3570
|
|
4
|
+
aml/hardrules.py,sha256=EH88y3G_gNPYNm08XDkMnndG_Tqd9ha-bzLDYHDIFfk,1700
|
|
5
|
+
aml/ledger.py,sha256=6TRsjA8iEzeb7xN1uzWPVTFWZC5XF40uNspV4Y2yusY,2718
|
|
6
|
+
aml/lock.py,sha256=UsQgOM2M5t1TaCWLGmc-YkoN4mnCqMLVzdPlS8UBzYs,1466
|
|
7
|
+
aml/recycle.py,sha256=OJ_Hcdo5pNhiVXszjW2ffDlHxPBVUHrm3N3aXtHcW9Y,4911
|
|
8
|
+
aml/search.py,sha256=zPwk4qLWLoO4bgbZeoyFVLcymd1N6rT_-xb8XP9C7Xs,1678
|
|
9
|
+
aml/security.py,sha256=hhNdVMAP26O63z4KBFGgVgi5oa2m9p0bHyV4lmIJqtY,3227
|
|
10
|
+
aml/store.py,sha256=eC8j-RrywYD85Ib2dfJ-byv7bj1E0bNlOD02DawRC0Q,6360
|
|
11
|
+
aml/util.py,sha256=F5ySjqmlzhfc_W4RU7qe2CNFJUMO3S25PGgfyTgEwd4,5216
|
|
12
|
+
aml/wal.py,sha256=_GmXllGFCc4SzowYnGTZaAMyw93RUnhYRxhF_wuhGYQ,3460
|
|
13
|
+
agent_audit_memory-0.2.0rc1.dist-info/METADATA,sha256=xfadDI2p3S1PKBj-60q-nHmyda9423dQTvjtQ5WT1Gs,12621
|
|
14
|
+
agent_audit_memory-0.2.0rc1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
15
|
+
agent_audit_memory-0.2.0rc1.dist-info/top_level.txt,sha256=AzDIX_UAV0wKYDYvnrRhOThAWX6JUwXUbo_OzdJSlOE,4
|
|
16
|
+
agent_audit_memory-0.2.0rc1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 agent-memory-ledger maintainers
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
aml
|
aml/__init__.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""agent-memory-ledger (aml): a verifiable memory layer for AI agents.
|
|
3
|
+
|
|
4
|
+
Invariants (fail-closed):
|
|
5
|
+
1. WAL before commit: every mutation is appended to the write-ahead log
|
|
6
|
+
first and can be replayed after a crash.
|
|
7
|
+
2. Triple pre-check: conflict / sensitive / redundant checks before write.
|
|
8
|
+
3. Deletion is soft: records go to the recycle bin (30 days, restorable).
|
|
9
|
+
4. Hash-chained trail: every entry links to the previous hash; verify()
|
|
10
|
+
detects any tamper offline.
|
|
11
|
+
5. hard rules never decay, archive, or get filtered.
|
|
12
|
+
6. Write lock: subagents may only write to draft; master promotes to core.
|
|
13
|
+
7. Zero sensitive payload persistence: only hashes / sanitized values.
|
|
14
|
+
8. Conflicts are surfaced in a ledger for human resolution, never merged
|
|
15
|
+
silently.
|
|
16
|
+
9. Single-writer enforcement: an exclusive cross-process lock guards each
|
|
17
|
+
storage directory (fail-closed multi-process contention).
|
|
18
|
+
10. Provable erasure: purge writes an erasure certificate (digest only) and
|
|
19
|
+
removes plaintext (GDPR Art.17 spot).
|
|
20
|
+
|
|
21
|
+
Research prototype / AI-assisted development. MIT licensed.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from .wal import WalLog
|
|
25
|
+
from .chain import HashChain
|
|
26
|
+
from .store import MemoryStore, LAYERS
|
|
27
|
+
from .recycle import RecycleBin, DEFAULT_RETENTION
|
|
28
|
+
from .ledger import ConflictLedger
|
|
29
|
+
from .lock import WriteLock, assert_core_write
|
|
30
|
+
from .hardrules import HardRules
|
|
31
|
+
from .search import MemorySearch
|
|
32
|
+
from .security import digest, sanitize, backup_dir, AuditLog
|
|
33
|
+
from .util import FileLock, ProcessContentionError, atomic_write
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"WalLog",
|
|
37
|
+
"HashChain",
|
|
38
|
+
"MemoryStore",
|
|
39
|
+
"LAYERS",
|
|
40
|
+
"RecycleBin",
|
|
41
|
+
"DEFAULT_RETENTION",
|
|
42
|
+
"ConflictLedger",
|
|
43
|
+
"WriteLock",
|
|
44
|
+
"assert_core_write",
|
|
45
|
+
"HardRules",
|
|
46
|
+
"MemorySearch",
|
|
47
|
+
"digest",
|
|
48
|
+
"sanitize",
|
|
49
|
+
"backup_dir",
|
|
50
|
+
"AuditLog",
|
|
51
|
+
"FileLock",
|
|
52
|
+
"ProcessContentionError",
|
|
53
|
+
"atomic_write",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
__version__ = "0.2.0rc1"
|
aml/chain.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Hash-chained audit trail: append-only, offline-verifiable."""
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
from .util import atomic_write
|
|
10
|
+
|
|
11
|
+
GENESIS = "0" * 64
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class HashChain:
|
|
15
|
+
"""Each record stores (lsn, payload, prev_hash, hash). verify() recomputes.
|
|
16
|
+
|
|
17
|
+
Ordering is defined by the monotonically increasing `lsn` sequence, not by
|
|
18
|
+
wall-clock time. System clock changes (including rollback) cannot reorder
|
|
19
|
+
or break the chain because each link includes the previous hash.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, path):
|
|
23
|
+
self.path = path
|
|
24
|
+
self._lock = threading.Lock()
|
|
25
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
26
|
+
|
|
27
|
+
def _last_hash(self):
|
|
28
|
+
last = None
|
|
29
|
+
if os.path.exists(self.path):
|
|
30
|
+
with open(self.path, encoding="utf-8") as f:
|
|
31
|
+
for line in f:
|
|
32
|
+
line = line.strip()
|
|
33
|
+
if line:
|
|
34
|
+
last = json.loads(line)
|
|
35
|
+
return last["hash"] if last else GENESIS
|
|
36
|
+
|
|
37
|
+
def _count(self):
|
|
38
|
+
if not os.path.exists(self.path):
|
|
39
|
+
return 0
|
|
40
|
+
with open(self.path, encoding="utf-8") as f:
|
|
41
|
+
return sum(1 for _ in f)
|
|
42
|
+
|
|
43
|
+
def append(self, payload):
|
|
44
|
+
"""Append one item. Returns the stored record dict."""
|
|
45
|
+
with self._lock:
|
|
46
|
+
prev = self._last_hash()
|
|
47
|
+
ser = json.dumps(payload, sort_keys=True, ensure_ascii=False)
|
|
48
|
+
digest = hashlib.sha256((ser + "|" + prev).encode("utf-8")).hexdigest()
|
|
49
|
+
lsn = self._count()
|
|
50
|
+
rec = {
|
|
51
|
+
"lsn": lsn,
|
|
52
|
+
"index": lsn, # legacy alias, same value
|
|
53
|
+
"ts": time.time(),
|
|
54
|
+
"payload": payload,
|
|
55
|
+
"prev_hash": prev,
|
|
56
|
+
"hash": digest,
|
|
57
|
+
}
|
|
58
|
+
with open(self.path, "a", encoding="utf-8") as f:
|
|
59
|
+
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
60
|
+
f.flush()
|
|
61
|
+
return rec
|
|
62
|
+
|
|
63
|
+
def verify(self):
|
|
64
|
+
"""Return (ok, first_bad_index_or_None, total_records)."""
|
|
65
|
+
prev = GENESIS
|
|
66
|
+
total = 0
|
|
67
|
+
if not os.path.exists(self.path):
|
|
68
|
+
return True, None, 0
|
|
69
|
+
with open(self.path, encoding="utf-8") as f:
|
|
70
|
+
for i, line in enumerate(f):
|
|
71
|
+
line = line.strip()
|
|
72
|
+
if not line:
|
|
73
|
+
continue
|
|
74
|
+
try:
|
|
75
|
+
rec = json.loads(line)
|
|
76
|
+
except Exception:
|
|
77
|
+
return False, i, total
|
|
78
|
+
ser = json.dumps(rec["payload"], sort_keys=True, ensure_ascii=False)
|
|
79
|
+
digest = hashlib.sha256((ser + "|" + prev).encode("utf-8")).hexdigest()
|
|
80
|
+
lsn = rec.get("lsn", rec.get("index"))
|
|
81
|
+
if (
|
|
82
|
+
rec.get("prev_hash") != prev
|
|
83
|
+
or rec.get("hash") != digest
|
|
84
|
+
or lsn != i
|
|
85
|
+
):
|
|
86
|
+
return False, i, total
|
|
87
|
+
prev = rec["hash"]
|
|
88
|
+
total = i + 1
|
|
89
|
+
return True, None, total
|
|
90
|
+
|
|
91
|
+
def tamper(self, index, new_payload):
|
|
92
|
+
"""Rewrite a record's payload WITHOUT updating its hash (for tamper tests)."""
|
|
93
|
+
with self._lock:
|
|
94
|
+
with open(self.path, encoding="utf-8") as f:
|
|
95
|
+
lines = f.read().splitlines()
|
|
96
|
+
rec = json.loads(lines[index])
|
|
97
|
+
rec["payload"] = new_payload # hash left stale -> verify must fail
|
|
98
|
+
lines[index] = json.dumps(rec, ensure_ascii=False)
|
|
99
|
+
atomic_write(self.path, "\n".join(lines) + "\n")
|
aml/hardrules.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Permanent rigid rules (hard rules): exempt from decay/archive/filtering."""
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class HardRules:
|
|
8
|
+
"""Keys marked hard are never decayed, archived, or filtered."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, rules=None, path=None):
|
|
11
|
+
self.rules = dict(rules or {})
|
|
12
|
+
if path and os.path.exists(path):
|
|
13
|
+
with open(path, encoding="utf-8") as f:
|
|
14
|
+
data = json.load(f)
|
|
15
|
+
for k, v in data.items():
|
|
16
|
+
self.rules[k] = v
|
|
17
|
+
|
|
18
|
+
def is_hard(self, key):
|
|
19
|
+
return self.rules.get(key, {}).get("hard", True) if key in self.rules else False
|
|
20
|
+
|
|
21
|
+
def register(self, key, **attrs):
|
|
22
|
+
self.rules[key] = {"hard": True, **attrs}
|
|
23
|
+
|
|
24
|
+
def decay(self, entries, factor=0.9):
|
|
25
|
+
"""Apply score decay to non-hard entries. Hard entries unchanged."""
|
|
26
|
+
out = []
|
|
27
|
+
for e in entries:
|
|
28
|
+
key = e.get("key") or e.get("name")
|
|
29
|
+
if self.is_hard(key):
|
|
30
|
+
out.append(e)
|
|
31
|
+
continue
|
|
32
|
+
e2 = dict(e)
|
|
33
|
+
e2["score"] = e.get("score", 1.0) * factor
|
|
34
|
+
out.append(e2)
|
|
35
|
+
return out
|
|
36
|
+
|
|
37
|
+
def archive_gate(self, entries, threshold=0.5):
|
|
38
|
+
"""Keep hard rules always; keep others only above score threshold."""
|
|
39
|
+
kept = []
|
|
40
|
+
for e in entries:
|
|
41
|
+
key = e.get("key") or e.get("name")
|
|
42
|
+
if self.is_hard(key) or e.get("score", 1.0) >= threshold:
|
|
43
|
+
kept.append(e)
|
|
44
|
+
return kept
|
|
45
|
+
|
|
46
|
+
def filter_out_hard(self, entries):
|
|
47
|
+
"""Return non-hard entries only (hard ones are always retained elsewhere)."""
|
|
48
|
+
return [e for e in entries if not self.is_hard(e.get("key") or e.get("name"))]
|
aml/ledger.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Conflict ledger: surface conflicting writes for human resolution.
|
|
3
|
+
|
|
4
|
+
Conflicts are never merged silently. A pending conflict blocks the write
|
|
5
|
+
until a human resolves it (resolve()).
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ConflictLedger:
|
|
14
|
+
"""Records conflicting writes as pending; resolve() is the human decision."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, path):
|
|
17
|
+
self.path = path
|
|
18
|
+
self._lock = threading.Lock()
|
|
19
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
20
|
+
self.entries = self._load()
|
|
21
|
+
|
|
22
|
+
def _load(self):
|
|
23
|
+
if os.path.exists(self.path):
|
|
24
|
+
with open(self.path, encoding="utf-8") as f:
|
|
25
|
+
return json.load(f)
|
|
26
|
+
return {"pending": [], "resolved": []}
|
|
27
|
+
|
|
28
|
+
def _save(self):
|
|
29
|
+
with open(self.path, "w", encoding="utf-8") as f:
|
|
30
|
+
json.dump(self.entries, f, ensure_ascii=False, indent=2)
|
|
31
|
+
|
|
32
|
+
def detect(self, key, incoming, current, source):
|
|
33
|
+
"""Return a conflict id if incoming conflicts with current, else None.
|
|
34
|
+
|
|
35
|
+
Same-source updates are treated as intentional (no conflict).
|
|
36
|
+
"""
|
|
37
|
+
if current is None:
|
|
38
|
+
return None
|
|
39
|
+
if current.get("payload") == incoming:
|
|
40
|
+
return None
|
|
41
|
+
cur_source = current.get("meta", {}).get("source")
|
|
42
|
+
if cur_source == source:
|
|
43
|
+
return None
|
|
44
|
+
entry = {
|
|
45
|
+
"id": "CFL-%d" % int(time.time() * 1000),
|
|
46
|
+
"key": key,
|
|
47
|
+
"incoming": incoming,
|
|
48
|
+
"current": current,
|
|
49
|
+
"source": source,
|
|
50
|
+
"status": "pending",
|
|
51
|
+
"ts": time.time(),
|
|
52
|
+
}
|
|
53
|
+
with self._lock:
|
|
54
|
+
self.entries["pending"].append(entry)
|
|
55
|
+
self._save()
|
|
56
|
+
return entry["id"]
|
|
57
|
+
|
|
58
|
+
def list_pending(self):
|
|
59
|
+
return list(self.entries["pending"])
|
|
60
|
+
|
|
61
|
+
def has_pending(self, key=None):
|
|
62
|
+
if key is None:
|
|
63
|
+
return bool(self.entries["pending"])
|
|
64
|
+
return any(e["key"] == key for e in self.entries["pending"])
|
|
65
|
+
|
|
66
|
+
def resolve(self, cid, decision, by="human"):
|
|
67
|
+
"""Resolve a pending conflict with a human decision (keep/incoming/merge)."""
|
|
68
|
+
with self._lock:
|
|
69
|
+
for i, e in enumerate(self.entries["pending"]):
|
|
70
|
+
if e["id"] == cid:
|
|
71
|
+
e["status"] = "resolved"
|
|
72
|
+
e["decision"] = decision
|
|
73
|
+
e["resolved_by"] = by
|
|
74
|
+
e["resolved_at"] = time.time()
|
|
75
|
+
self.entries["resolved"].append(e)
|
|
76
|
+
del self.entries["pending"][i]
|
|
77
|
+
self._save()
|
|
78
|
+
return e
|
|
79
|
+
raise KeyError(cid)
|
|
80
|
+
|
|
81
|
+
def list_resolved(self):
|
|
82
|
+
return list(self.entries["resolved"])
|
aml/lock.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Write lock & role-based write permission.
|
|
3
|
+
|
|
4
|
+
Rule: subagents may only write to draft; master may promote to core.
|
|
5
|
+
Any direct core write without master role is denied (fail-closed).
|
|
6
|
+
"""
|
|
7
|
+
import threading
|
|
8
|
+
|
|
9
|
+
ROLES = ("subagent", "master")
|
|
10
|
+
WRITABLE_LAYERS = {"subagent": {"draft"}, "master": {"draft", "hot", "core", "archive"}}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class WriteLock:
|
|
14
|
+
"""Mutex for promoting a draft into core (single-writer)."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, owner="master"):
|
|
17
|
+
self.owner = owner
|
|
18
|
+
self._lock = threading.Lock()
|
|
19
|
+
self._held_by = None
|
|
20
|
+
|
|
21
|
+
def acquire(self, who):
|
|
22
|
+
if self._held_by is not None:
|
|
23
|
+
raise PermissionError("write lock already held by %s" % self._held_by)
|
|
24
|
+
with self._lock:
|
|
25
|
+
if self._held_by is not None:
|
|
26
|
+
raise PermissionError("write lock already held by %s" % self._held_by)
|
|
27
|
+
self._held_by = who
|
|
28
|
+
return True
|
|
29
|
+
|
|
30
|
+
def release(self):
|
|
31
|
+
self._held_by = None
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def held_by(self):
|
|
35
|
+
return self._held_by
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def assert_core_write(role):
|
|
39
|
+
"""Deny unless the role is master (fail-closed)."""
|
|
40
|
+
if role not in ROLES:
|
|
41
|
+
raise PermissionError("unknown role: %s" % role)
|
|
42
|
+
if "core" not in WRITABLE_LAYERS.get(role, set()):
|
|
43
|
+
raise PermissionError("role %s cannot write core" % role)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def can_write(role, layer):
|
|
47
|
+
"""Return True if role may write to layer."""
|
|
48
|
+
return layer in WRITABLE_LAYERS.get(role, set())
|
aml/recycle.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Recycle bin: soft deletion with 30-day retention and restore.
|
|
3
|
+
|
|
4
|
+
Provable erasure (expert review, GDPR Art.17 spot):
|
|
5
|
+
* purge_expired physically removes the plaintext record and writes an
|
|
6
|
+
**erasure certificate** (erased_at / erased_by / payload_digest) into
|
|
7
|
+
<base_dir>/erasure/certificates.jsonl. Only the digest is retained as
|
|
8
|
+
proof that the record existed; the plaintext payload is gone.
|
|
9
|
+
* The hash chain records a `purge` marker, so an offline auditor can
|
|
10
|
+
confirm the record existed and was erased without seeing its content.
|
|
11
|
+
"""
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import time
|
|
15
|
+
|
|
16
|
+
from .util import atomic_write
|
|
17
|
+
|
|
18
|
+
DEFAULT_RETENTION = 30 * 24 * 3600 # 30 days
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RecycleBin:
|
|
22
|
+
"""Deletion is soft: records move to recycle, restorable within retention."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, store, retention=DEFAULT_RETENTION, erasure_dir=None):
|
|
25
|
+
self.store = store
|
|
26
|
+
self.retention = retention
|
|
27
|
+
self.erasure_dir = erasure_dir or os.path.join(store.base_dir, "erasure")
|
|
28
|
+
os.makedirs(self.erasure_dir, exist_ok=True)
|
|
29
|
+
|
|
30
|
+
def recycle(self, layer, key, by="system"):
|
|
31
|
+
"""Soft-delete a record from a layer into the recycle bin."""
|
|
32
|
+
rec = self.store.records[layer].pop(key)
|
|
33
|
+
rec["__deleted_at"] = time.time()
|
|
34
|
+
rec["__from_layer"] = layer
|
|
35
|
+
rec["__deleted_by"] = by
|
|
36
|
+
self.store.records["recycle"][key] = rec
|
|
37
|
+
self.store._persist(layer)
|
|
38
|
+
self.store._persist("recycle")
|
|
39
|
+
self.store.wal.append({"op": "recycle", "key": key, "layer": layer})
|
|
40
|
+
self.store.chain.append({"op": "recycle", "key": key})
|
|
41
|
+
return rec
|
|
42
|
+
|
|
43
|
+
def restore(self, key):
|
|
44
|
+
"""Restore a recycled record back to its original layer (within retention)."""
|
|
45
|
+
rec = self.store.records["recycle"].pop(key)
|
|
46
|
+
layer = rec.pop("__from_layer")
|
|
47
|
+
rec.pop("__deleted_at", None)
|
|
48
|
+
rec.pop("__deleted_by", None)
|
|
49
|
+
self.store.records[layer][key] = rec
|
|
50
|
+
self.store._persist("recycle")
|
|
51
|
+
self.store._persist(layer)
|
|
52
|
+
self.store.wal.append({"op": "restore", "key": key, "layer": layer})
|
|
53
|
+
return rec
|
|
54
|
+
|
|
55
|
+
def list_recycled(self, now=None):
|
|
56
|
+
now = now or time.time()
|
|
57
|
+
out = []
|
|
58
|
+
for key, rec in self.store.records["recycle"].items():
|
|
59
|
+
deleted_at = rec.get("__deleted_at", 0)
|
|
60
|
+
age = now - deleted_at
|
|
61
|
+
out.append(
|
|
62
|
+
{
|
|
63
|
+
"key": key,
|
|
64
|
+
"from_layer": rec.get("__from_layer"),
|
|
65
|
+
"deleted_at": deleted_at,
|
|
66
|
+
"expired": age >= self.retention,
|
|
67
|
+
"retain_secs_left": max(0, self.retention - age),
|
|
68
|
+
}
|
|
69
|
+
)
|
|
70
|
+
return out
|
|
71
|
+
|
|
72
|
+
def purge_expired(self, now=None, erased_by="system"):
|
|
73
|
+
"""Physically erase expired entries; returns a list of erasure certificates.
|
|
74
|
+
|
|
75
|
+
Each certificate retains only the payload digest (proof of existence),
|
|
76
|
+
never the plaintext. Append-only JSONL under `<base>/erasure/`.
|
|
77
|
+
"""
|
|
78
|
+
now = now or time.time()
|
|
79
|
+
certs = []
|
|
80
|
+
for key in list(self.store.records["recycle"].keys()):
|
|
81
|
+
rec = self.store.records["recycle"][key]
|
|
82
|
+
if (now - rec.get("__deleted_at", 0)) >= self.retention:
|
|
83
|
+
cert = {
|
|
84
|
+
"key": key,
|
|
85
|
+
"from_layer": rec.get("__from_layer"),
|
|
86
|
+
"erased_at": now,
|
|
87
|
+
"erased_by": erased_by,
|
|
88
|
+
"payload_digest": rec.get("hash"),
|
|
89
|
+
"retention_seconds": self.retention,
|
|
90
|
+
}
|
|
91
|
+
self.store.records["recycle"].pop(key)
|
|
92
|
+
self.store._persist("recycle")
|
|
93
|
+
self.store.wal.append({"op": "purge", "key": key, "erased_at": now})
|
|
94
|
+
self.store.chain.append(
|
|
95
|
+
{"op": "purge", "key": key, "payload_digest": cert["payload_digest"]}
|
|
96
|
+
)
|
|
97
|
+
self._write_certificate(cert)
|
|
98
|
+
certs.append(cert)
|
|
99
|
+
return certs
|
|
100
|
+
|
|
101
|
+
def _write_certificate(self, cert):
|
|
102
|
+
# canonical JSON: sort_keys so any machine produces the identical
|
|
103
|
+
# byte string; a digest-verifiable certificate format (expert #7).
|
|
104
|
+
line = json.dumps(cert, sort_keys=True, ensure_ascii=False)
|
|
105
|
+
path = os.path.join(self.erasure_dir, "certificates.jsonl")
|
|
106
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
107
|
+
f.write(line + "\n")
|
|
108
|
+
f.flush()
|
|
109
|
+
|
|
110
|
+
def read_certificates(self):
|
|
111
|
+
"""Read erasure certificates (audit-side helper)."""
|
|
112
|
+
path = os.path.join(self.erasure_dir, "certificates.jsonl")
|
|
113
|
+
if not os.path.exists(path):
|
|
114
|
+
return []
|
|
115
|
+
out = []
|
|
116
|
+
with open(path, encoding="utf-8") as f:
|
|
117
|
+
for line in f:
|
|
118
|
+
line = line.strip()
|
|
119
|
+
if line:
|
|
120
|
+
out.append(json.loads(line))
|
|
121
|
+
return out
|
aml/search.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Layered retrieval: mandatory hard rules -> relevance -> cleanup -> receipt."""
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class MemorySearch:
|
|
7
|
+
"""Query across layers. Hard rules are surfaced first and never filtered."""
|
|
8
|
+
|
|
9
|
+
def __init__(self, store, hard_rules):
|
|
10
|
+
self.store = store
|
|
11
|
+
self.hard = hard_rules
|
|
12
|
+
|
|
13
|
+
@staticmethod
|
|
14
|
+
def _score(text, query):
|
|
15
|
+
q = query.lower()
|
|
16
|
+
t = str(text).lower()
|
|
17
|
+
words = re.findall(r"[\w\u4e00-\u9fff]+", q)
|
|
18
|
+
if not words:
|
|
19
|
+
return 0.0
|
|
20
|
+
return sum(1 for w in words if w in t) / len(words)
|
|
21
|
+
|
|
22
|
+
def retrieve(self, query, layers=("core", "hot"), k=10):
|
|
23
|
+
"""Return scored results. Each has {key, layer, score, hard, source}.
|
|
24
|
+
|
|
25
|
+
Step 1: mandatory (hard rules first). Step 2: relevance.
|
|
26
|
+
Step 3: cleanup (recycle-bin / expired are excluded upstream).
|
|
27
|
+
Step 4: receipt (source annotated).
|
|
28
|
+
"""
|
|
29
|
+
results = []
|
|
30
|
+
for layer in layers:
|
|
31
|
+
for key, rec in self.store.records.get(layer, {}).items():
|
|
32
|
+
text = str(rec.get("payload", "")) + " " + str(rec.get("meta", {}))
|
|
33
|
+
score = self._score(text, query)
|
|
34
|
+
if score > 0:
|
|
35
|
+
results.append(
|
|
36
|
+
{
|
|
37
|
+
"key": key,
|
|
38
|
+
"layer": layer,
|
|
39
|
+
"score": score,
|
|
40
|
+
"hard": self.hard.is_hard(key),
|
|
41
|
+
"source": rec.get("meta", {}).get("source", "n/a"),
|
|
42
|
+
}
|
|
43
|
+
)
|
|
44
|
+
results.sort(key=lambda r: (r["hard"], r["score"]), reverse=True)
|
|
45
|
+
return results[:k]
|
aml/security.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Security helpers: digest, sanitize, backups, audit events."""
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import shutil
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
# Default patterns used by sanitize() to redact likely secrets.
|
|
11
|
+
# Label prefixes are non-capturing so the whole match becomes [REDACTED].
|
|
12
|
+
DEFAULT_PATTERNS = [
|
|
13
|
+
r"(?:password\s*[:=]\s*)\S+",
|
|
14
|
+
r"(?:passwd\s*[:=]\s*)\S+",
|
|
15
|
+
r"(?:token\s*[:=]\s*)\S+",
|
|
16
|
+
r"(?:api[_-]?key\s*[:=]\s*)\S+",
|
|
17
|
+
r"(?:secret\s*[:=]\s*)\S+",
|
|
18
|
+
r"(?:authorization\s*[:=]\s*)\S+",
|
|
19
|
+
r"[A-Za-z0-9+/]{40,}={0,2}",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
# Dict keys that force the whole value to be redacted regardless of shape.
|
|
23
|
+
SENSITIVE_KEY_TOKENS = (
|
|
24
|
+
"password", "passwd", "token", "secret",
|
|
25
|
+
"api_key", "apikey", "authorization", "auth", "private_key",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def digest(obj):
|
|
30
|
+
"""Stable sha256 of a JSON-serializable object."""
|
|
31
|
+
ser = json.dumps(obj, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
32
|
+
return hashlib.sha256(ser).hexdigest()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def sanitize(obj, patterns=None, sensitive_key_tokens=None):
|
|
36
|
+
"""Recursively redact secret-like values. Never store raw secrets.
|
|
37
|
+
|
|
38
|
+
- Strings matching secret patterns are replaced with [REDACTED].
|
|
39
|
+
- Dict values under sensitive keys (password/token/secret/api_key/...)
|
|
40
|
+
are redacted even if their shape does not match a pattern.
|
|
41
|
+
"""
|
|
42
|
+
patterns = patterns or DEFAULT_PATTERNS
|
|
43
|
+
sensitive_key_tokens = sensitive_key_tokens or SENSITIVE_KEY_TOKENS
|
|
44
|
+
if isinstance(obj, str):
|
|
45
|
+
s = obj
|
|
46
|
+
for p in patterns:
|
|
47
|
+
s = re.sub(p, "[REDACTED]", s, flags=re.I)
|
|
48
|
+
return s
|
|
49
|
+
if isinstance(obj, list):
|
|
50
|
+
return [sanitize(x, patterns, sensitive_key_tokens) for x in obj]
|
|
51
|
+
if isinstance(obj, dict):
|
|
52
|
+
out = {}
|
|
53
|
+
for k, v in obj.items():
|
|
54
|
+
klow = str(k).lower()
|
|
55
|
+
if any(tok in klow for tok in sensitive_key_tokens):
|
|
56
|
+
out[k] = sanitize(v, patterns, sensitive_key_tokens) if isinstance(v, (dict, list)) else "[REDACTED]"
|
|
57
|
+
else:
|
|
58
|
+
out[k] = sanitize(v, patterns, sensitive_key_tokens)
|
|
59
|
+
return out
|
|
60
|
+
return obj
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def hash_only(obj):
|
|
64
|
+
"""Return only the digest of an object (never the object itself)."""
|
|
65
|
+
return digest(obj)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def backup_dir(src, backup_root, tag=""):
|
|
69
|
+
"""Copy a directory into backup_root with a timestamp. Returns dst path."""
|
|
70
|
+
ts = time.strftime("%Y%m%d_%H%M%S")
|
|
71
|
+
name = os.path.basename(src.rstrip("\\/"))
|
|
72
|
+
dst = os.path.join(backup_root, f"{name}_{tag}_{ts}" if tag else f"{name}_{ts}")
|
|
73
|
+
shutil.copytree(src, dst)
|
|
74
|
+
return dst
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class AuditLog:
|
|
78
|
+
"""Append-only JSONL audit event log."""
|
|
79
|
+
|
|
80
|
+
def __init__(self, path):
|
|
81
|
+
self.path = path
|
|
82
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
83
|
+
|
|
84
|
+
def record(self, event):
|
|
85
|
+
with open(self.path, "a", encoding="utf-8") as f:
|
|
86
|
+
f.write(json.dumps({"ts": time.time(), **event}, ensure_ascii=False) + "\n")
|
|
87
|
+
|
|
88
|
+
def read(self):
|
|
89
|
+
if not os.path.exists(self.path):
|
|
90
|
+
return []
|
|
91
|
+
out = []
|
|
92
|
+
with open(self.path, encoding="utf-8") as f:
|
|
93
|
+
for line in f:
|
|
94
|
+
line = line.strip()
|
|
95
|
+
if line:
|
|
96
|
+
out.append(json.loads(line))
|
|
97
|
+
return out
|
aml/store.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Layered memory store: draft / hot / core / archive, WAL-protected."""
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from .chain import HashChain
|
|
8
|
+
from .security import digest, sanitize
|
|
9
|
+
from .util import atomic_write, try_exclusive_lock
|
|
10
|
+
from .wal import WalLog
|
|
11
|
+
|
|
12
|
+
LAYERS = ("draft", "hot", "core", "archive")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class MemoryStore:
|
|
16
|
+
"""A small layered store. Every put goes through WAL then hash-chain.
|
|
17
|
+
|
|
18
|
+
Concurrency contract (expert review, pre-release):
|
|
19
|
+
* v0.2 enforces a **single-process writer** per storage directory via an
|
|
20
|
+
exclusive cross-process file lock (fail-closed). A second process
|
|
21
|
+
attempting to write raises ProcessContentionError instead of silently
|
|
22
|
+
corrupting files.
|
|
23
|
+
* Within one process, methods are thread-safe for reads; writes are
|
|
24
|
+
serialized by the WAL/chain locks.
|
|
25
|
+
* Pass `exclusive=False` only when you explicitly share the directory
|
|
26
|
+
from another trusted process (not recommended before SQLite backend).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, base_dir, fsync=False, durability=None, exclusive=True, namespace=None):
|
|
30
|
+
self.base_dir = base_dir
|
|
31
|
+
self.namespace = namespace
|
|
32
|
+
os.makedirs(base_dir, exist_ok=True)
|
|
33
|
+
if namespace:
|
|
34
|
+
# Physical isolation per tenant/session namespace (invariant #9).
|
|
35
|
+
base_dir = os.path.join(base_dir, "namespaces", namespace)
|
|
36
|
+
os.makedirs(base_dir, exist_ok=True)
|
|
37
|
+
self.base_dir = base_dir
|
|
38
|
+
|
|
39
|
+
self._exclusive_lock = None
|
|
40
|
+
if exclusive:
|
|
41
|
+
self._exclusive_lock = try_exclusive_lock(
|
|
42
|
+
os.path.join(self.base_dir, "aml.lock"), who="writer"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
self._dirs = {}
|
|
46
|
+
for layer in LAYERS:
|
|
47
|
+
d = os.path.join(base_dir, layer)
|
|
48
|
+
os.makedirs(d, exist_ok=True)
|
|
49
|
+
self._dirs[layer] = d
|
|
50
|
+
os.makedirs(os.path.join(base_dir, "recycle"), exist_ok=True)
|
|
51
|
+
os.makedirs(os.path.join(base_dir, "wal"), exist_ok=True)
|
|
52
|
+
os.makedirs(os.path.join(base_dir, "chain"), exist_ok=True)
|
|
53
|
+
|
|
54
|
+
self.wal = WalLog(os.path.join(base_dir, "wal", "wal.log"), fsync=fsync, durability=durability)
|
|
55
|
+
self.chain = HashChain(os.path.join(base_dir, "chain", "chain.jsonl"))
|
|
56
|
+
self.records = {layer: self._load(layer) for layer in LAYERS}
|
|
57
|
+
self.records["recycle"] = self._load("recycle")
|
|
58
|
+
|
|
59
|
+
def close(self):
|
|
60
|
+
"""Release the exclusive cross-process lock (if held). Safe to call twice."""
|
|
61
|
+
if self._exclusive_lock is not None:
|
|
62
|
+
try:
|
|
63
|
+
self._exclusive_lock.release()
|
|
64
|
+
finally:
|
|
65
|
+
self._exclusive_lock = None
|
|
66
|
+
|
|
67
|
+
def __enter__(self):
|
|
68
|
+
return self
|
|
69
|
+
|
|
70
|
+
def __exit__(self, *exc):
|
|
71
|
+
self.close()
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
def __del__(self):
|
|
75
|
+
try:
|
|
76
|
+
self.close()
|
|
77
|
+
except Exception:
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
def _data_path(self, layer):
|
|
81
|
+
return os.path.join(self.base_dir, layer, "data.json")
|
|
82
|
+
|
|
83
|
+
def _load(self, layer):
|
|
84
|
+
p = self._data_path(layer)
|
|
85
|
+
if not os.path.exists(p):
|
|
86
|
+
return {}
|
|
87
|
+
with open(p, encoding="utf-8") as f:
|
|
88
|
+
return json.load(f)
|
|
89
|
+
|
|
90
|
+
def _persist(self, layer):
|
|
91
|
+
atomic_write(
|
|
92
|
+
self._data_path(layer),
|
|
93
|
+
json.dumps(self.records[layer], ensure_ascii=False, indent=2),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# -- API -----------------------------------------------------------------
|
|
97
|
+
def put(self, record, source="system"):
|
|
98
|
+
"""Write one record. record = {layer,key,payload,meta}. Returns rec_id."""
|
|
99
|
+
layer = record.get("layer")
|
|
100
|
+
key = record.get("key")
|
|
101
|
+
if layer not in LAYERS:
|
|
102
|
+
raise ValueError("bad layer: %s" % layer)
|
|
103
|
+
if not key:
|
|
104
|
+
raise ValueError("empty key")
|
|
105
|
+
payload = record.get("payload")
|
|
106
|
+
payload_hash = digest(payload)
|
|
107
|
+
meta = dict(record.get("meta") or {})
|
|
108
|
+
meta.setdefault("source", source)
|
|
109
|
+
|
|
110
|
+
# invariant #7: zero sensitive persistence. By default the payload is
|
|
111
|
+
# sanitized (secrets redacted / hashed) BEFORE storage, so raw secrets
|
|
112
|
+
# never hit disk. Callers that genuinely need raw payloads may pass
|
|
113
|
+
# sanitize=False explicitly, but must then own the data-classification.
|
|
114
|
+
safe_payload = sanitize(payload) if record.get("sanitize", True) else payload
|
|
115
|
+
rec_id = self.wal.append(
|
|
116
|
+
{"op": "put", "layer": layer, "key": key, "payload_hash": payload_hash, "ts": time.time()}
|
|
117
|
+
)
|
|
118
|
+
self.records[layer][key] = {
|
|
119
|
+
"payload": safe_payload,
|
|
120
|
+
"meta": meta,
|
|
121
|
+
"hash": payload_hash,
|
|
122
|
+
"rec_id": rec_id,
|
|
123
|
+
"ts": time.time(),
|
|
124
|
+
}
|
|
125
|
+
self._persist(layer)
|
|
126
|
+
self.chain.append({"op": "put", "layer": layer, "key": key, "payload_hash": payload_hash})
|
|
127
|
+
return rec_id
|
|
128
|
+
|
|
129
|
+
def get(self, layer, key, default=None):
|
|
130
|
+
return self.records.get(layer, {}).get(key, default)
|
|
131
|
+
|
|
132
|
+
def exists(self, layer, key):
|
|
133
|
+
return key in self.records.get(layer, {})
|
|
134
|
+
|
|
135
|
+
def list_layer(self, layer):
|
|
136
|
+
return list(self.records.get(layer, {}).keys())
|
|
137
|
+
|
|
138
|
+
def commit_draft(self, key, role="subagent"):
|
|
139
|
+
"""Promote a draft record to core. Requires master role (write lock)."""
|
|
140
|
+
from .lock import assert_core_write
|
|
141
|
+
|
|
142
|
+
assert_core_write(role)
|
|
143
|
+
rec = self.records["draft"].pop(key)
|
|
144
|
+
self.records["core"][key] = rec
|
|
145
|
+
self._persist("draft")
|
|
146
|
+
self._persist("core")
|
|
147
|
+
self.wal.append({"op": "promote", "key": key, "layer": "core"})
|
|
148
|
+
self.chain.append({"op": "promote", "key": key})
|
|
149
|
+
return rec
|
|
150
|
+
|
|
151
|
+
def promote(self, key, to_layer, role="subagent"):
|
|
152
|
+
"""Generic layer promotion (draft->hot->core etc). Master only to core."""
|
|
153
|
+
from .lock import assert_core_write
|
|
154
|
+
|
|
155
|
+
for layer in ("draft", "hot", "core"):
|
|
156
|
+
if self.exists(layer, key):
|
|
157
|
+
rec = self.records[layer].pop(key)
|
|
158
|
+
if to_layer == "core":
|
|
159
|
+
assert_core_write(role)
|
|
160
|
+
self.records[to_layer][key] = rec
|
|
161
|
+
self._persist(layer)
|
|
162
|
+
self._persist(to_layer)
|
|
163
|
+
self.wal.append({"op": "promote", "key": key, "layer": to_layer})
|
|
164
|
+
self.chain.append({"op": "promote", "key": key, "to": to_layer})
|
|
165
|
+
return rec
|
|
166
|
+
raise KeyError(key)
|
aml/util.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Zero-dependency process-level utilities: atomic writes + cross-process lock.
|
|
3
|
+
|
|
4
|
+
Why this file exists (expert review, pre-release):
|
|
5
|
+
* Multi-process writers against the same JSON/WAL files corrupt state
|
|
6
|
+
(file tearing, broken hash chain). Thread locks are not enough.
|
|
7
|
+
* Plain writes can leave a torn file after a power loss; atomic
|
|
8
|
+
write-then-rename makes a JSON file either fully old or fully new.
|
|
9
|
+
|
|
10
|
+
Design:
|
|
11
|
+
* atomic_write: write to a temp file in the same directory, fsync, then
|
|
12
|
+
os.replace() over the target. Readers never observe a half-written file.
|
|
13
|
+
* FileLock: best-effort exclusive lock via msvcrt (Windows) or fcntl
|
|
14
|
+
(POSIX). No third-party dependency. Used by MemoryStore to enforce
|
|
15
|
+
single-process writer semantics per storage directory.
|
|
16
|
+
"""
|
|
17
|
+
import errno
|
|
18
|
+
import os
|
|
19
|
+
import tempfile
|
|
20
|
+
|
|
21
|
+
try: # Windows
|
|
22
|
+
import msvcrt
|
|
23
|
+
|
|
24
|
+
_LOCK_IMPL = "msvcrt"
|
|
25
|
+
except ImportError: # POSIX
|
|
26
|
+
import fcntl
|
|
27
|
+
|
|
28
|
+
_LOCK_IMPL = "fcntl"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class AtomicWriteError(OSError):
|
|
32
|
+
"""Raised when an atomic write fails (disk full, permission, rename)."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def atomic_write(path, data, encoding="utf-8"):
|
|
36
|
+
"""Atomically replace `path` with `data` (string).
|
|
37
|
+
|
|
38
|
+
Raises:
|
|
39
|
+
AtomicWriteError — with a distinct message for disk-full / permission /
|
|
40
|
+
unsupported-rename cases, so callers can surface a helpful error.
|
|
41
|
+
"""
|
|
42
|
+
directory = os.path.dirname(path) or "."
|
|
43
|
+
fd, tmp = tempfile.mkstemp(prefix=".aml_tmp_", dir=directory)
|
|
44
|
+
try:
|
|
45
|
+
with os.fdopen(fd, "w", encoding=encoding) as f:
|
|
46
|
+
f.write(data)
|
|
47
|
+
f.flush()
|
|
48
|
+
os.fsync(f.fileno())
|
|
49
|
+
try:
|
|
50
|
+
os.replace(tmp, path)
|
|
51
|
+
except OSError as e:
|
|
52
|
+
# e.g. NFS / network share without atomic rename support
|
|
53
|
+
raise AtomicWriteError(
|
|
54
|
+
"atomic rename failed for %s (is this a network/NFS share? "
|
|
55
|
+
"network shares are not recommended as aml storage): %s" % (path, e)
|
|
56
|
+
) from e
|
|
57
|
+
except OSError as e:
|
|
58
|
+
# disk full (ENOSPC) / permission (EACCES) / etc.
|
|
59
|
+
raise AtomicWriteError(
|
|
60
|
+
"atomic write failed for %s (%s). Check disk space and permissions." % (path, e)
|
|
61
|
+
) from e
|
|
62
|
+
finally:
|
|
63
|
+
# best-effort cleanup of the temp file so no *.aml_tmp_ garbage stays
|
|
64
|
+
if os.path.exists(tmp):
|
|
65
|
+
try:
|
|
66
|
+
os.unlink(tmp)
|
|
67
|
+
except OSError:
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class FileLock:
|
|
72
|
+
"""Exclusive advisory lock keyed to a lock file.
|
|
73
|
+
|
|
74
|
+
Usage::
|
|
75
|
+
|
|
76
|
+
lock = FileLock("mem/aml.lock")
|
|
77
|
+
with lock:
|
|
78
|
+
... # only one process at a time
|
|
79
|
+
|
|
80
|
+
Non-blocking by default; multi-process second writer raises OSError.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def __init__(self, path):
|
|
84
|
+
self.path = path
|
|
85
|
+
self._fd = None
|
|
86
|
+
|
|
87
|
+
def acquire(self, blocking=False):
|
|
88
|
+
if self._fd is not None:
|
|
89
|
+
return True
|
|
90
|
+
os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
|
|
91
|
+
self._fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o644)
|
|
92
|
+
try:
|
|
93
|
+
if _LOCK_IMPL == "msvcrt":
|
|
94
|
+
# Lock the whole file; non-blocking raises OSError.
|
|
95
|
+
msvcrt.locking(self._fd, msvcrt.LK_NBLCK, 1)
|
|
96
|
+
else:
|
|
97
|
+
flags = fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)
|
|
98
|
+
fcntl.flock(self._fd, flags)
|
|
99
|
+
return True
|
|
100
|
+
except OSError:
|
|
101
|
+
os.close(self._fd)
|
|
102
|
+
self._fd = None
|
|
103
|
+
raise
|
|
104
|
+
|
|
105
|
+
def release(self):
|
|
106
|
+
if self._fd is not None:
|
|
107
|
+
try:
|
|
108
|
+
if _LOCK_IMPL == "msvcrt":
|
|
109
|
+
msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1)
|
|
110
|
+
else:
|
|
111
|
+
fcntl.flock(self._fd, fcntl.LOCK_UN)
|
|
112
|
+
finally:
|
|
113
|
+
os.close(self._fd)
|
|
114
|
+
self._fd = None
|
|
115
|
+
|
|
116
|
+
def __enter__(self):
|
|
117
|
+
self.acquire(blocking=False)
|
|
118
|
+
return self
|
|
119
|
+
|
|
120
|
+
def __exit__(self, *exc):
|
|
121
|
+
self.release()
|
|
122
|
+
return False
|
|
123
|
+
|
|
124
|
+
def __del__(self):
|
|
125
|
+
# Best-effort release so OS file locks never leak (Windows deletes
|
|
126
|
+
# temp dirs only after all handles are closed).
|
|
127
|
+
try:
|
|
128
|
+
self.release()
|
|
129
|
+
except Exception:
|
|
130
|
+
pass
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class ProcessContentionError(OSError):
|
|
134
|
+
"""Another process already holds the exclusive store lock."""
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def try_exclusive_lock(path, who="writer"):
|
|
138
|
+
"""Acquire an exclusive lock or raise ProcessContentionError (fail-closed).
|
|
139
|
+
|
|
140
|
+
Returns a FileLock handle whose release() unlocks.
|
|
141
|
+
"""
|
|
142
|
+
lock = FileLock(path)
|
|
143
|
+
try:
|
|
144
|
+
lock.acquire(blocking=False)
|
|
145
|
+
except OSError as e:
|
|
146
|
+
if e.errno in (errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK, errno.EDEADLK):
|
|
147
|
+
raise ProcessContentionError(
|
|
148
|
+
"同一存储目录只允许一个写进程 (one writer per storage dir); "
|
|
149
|
+
"role=%s, lock=%s. If you need concurrent processes, give each "
|
|
150
|
+
"process its own namespace= or directory, or add your own "
|
|
151
|
+
"queueing/retry in the calling layer (the library deliberately "
|
|
152
|
+
"does NOT auto-retry, to preserve fail-closed semantics)."
|
|
153
|
+
% (who, path)
|
|
154
|
+
) from e
|
|
155
|
+
raise
|
|
156
|
+
return lock
|
aml/wal.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Write-ahead log: every mutation is appended first, replayable after crash."""
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
_FIELDS = ("op", "key", "layer", "payload_hash", "ts")
|
|
10
|
+
|
|
11
|
+
# Durability levels (expert review, pre-release):
|
|
12
|
+
# strict: fsync after every append. Power-loss safe; audit-grade guarantee.
|
|
13
|
+
# normal: flush to OS each append, fsync on batch boundaries (default).
|
|
14
|
+
# fast: no fsync; only process-exit safety. Testing only.
|
|
15
|
+
DURABILITY_LEVELS = ("strict", "normal", "fast")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _normalize_durability(value):
|
|
19
|
+
"""Accept new ('strict'|'normal'|'fast') or legacy bool (True->strict, False->normal)."""
|
|
20
|
+
if isinstance(value, bool):
|
|
21
|
+
return "strict" if value else "normal"
|
|
22
|
+
if value not in DURABILITY_LEVELS:
|
|
23
|
+
raise ValueError("durability must be one of %s" % (DURABILITY_LEVELS,))
|
|
24
|
+
return value
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class WalLog:
|
|
28
|
+
"""Append-only, integrity-checked write-ahead log (JSONL)."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, path, fsync=False, durability=None):
|
|
31
|
+
# legacy `fsync` kept for backward compatibility; `durability` wins.
|
|
32
|
+
self.durability = _normalize_durability(durability if durability is not None else fsync)
|
|
33
|
+
self.path = path
|
|
34
|
+
self._lock = threading.Lock()
|
|
35
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
36
|
+
|
|
37
|
+
@staticmethod
|
|
38
|
+
def _rec_id(rec):
|
|
39
|
+
body = json.dumps(
|
|
40
|
+
{k: rec[k] for k in _FIELDS if k in rec},
|
|
41
|
+
sort_keys=True,
|
|
42
|
+
ensure_ascii=False,
|
|
43
|
+
)
|
|
44
|
+
return hashlib.sha256(body.encode("utf-8")).hexdigest()[:16]
|
|
45
|
+
|
|
46
|
+
def append(self, entry):
|
|
47
|
+
"""Append one entry. Returns the rec_id (short hash)."""
|
|
48
|
+
with self._lock:
|
|
49
|
+
rec = dict(entry)
|
|
50
|
+
rec.setdefault("ts", time.time())
|
|
51
|
+
rec["rec_id"] = self._rec_id(rec)
|
|
52
|
+
line = json.dumps(rec, ensure_ascii=False)
|
|
53
|
+
with open(self.path, "a", encoding="utf-8") as f:
|
|
54
|
+
f.write(line + "\n")
|
|
55
|
+
f.flush()
|
|
56
|
+
if self.durability == "strict":
|
|
57
|
+
os.fsync(f.fileno())
|
|
58
|
+
return rec["rec_id"]
|
|
59
|
+
|
|
60
|
+
def read_all(self):
|
|
61
|
+
"""Return (valid_records, bad_records_count). Bad tail = crash truncation."""
|
|
62
|
+
recs, bad = [], 0
|
|
63
|
+
if not os.path.exists(self.path):
|
|
64
|
+
return recs, bad
|
|
65
|
+
with open(self.path, encoding="utf-8") as f:
|
|
66
|
+
for line in f:
|
|
67
|
+
line = line.strip()
|
|
68
|
+
if not line:
|
|
69
|
+
continue
|
|
70
|
+
try:
|
|
71
|
+
rec = json.loads(line)
|
|
72
|
+
if rec.get("rec_id") == self._rec_id(rec):
|
|
73
|
+
recs.append(rec)
|
|
74
|
+
else:
|
|
75
|
+
bad += 1
|
|
76
|
+
except Exception:
|
|
77
|
+
bad += 1
|
|
78
|
+
return recs, bad
|
|
79
|
+
|
|
80
|
+
def replay(self, apply_fn):
|
|
81
|
+
"""Replay all valid records through apply_fn. Returns (count, bad)."""
|
|
82
|
+
recs, bad = self.read_all()
|
|
83
|
+
for rec in recs:
|
|
84
|
+
apply_fn(rec)
|
|
85
|
+
return len(recs), bad
|
|
86
|
+
|
|
87
|
+
def truncate_last(self):
|
|
88
|
+
"""Simulate a crash: remove the final line (for tests / recovery drills)."""
|
|
89
|
+
with self._lock:
|
|
90
|
+
with open(self.path, encoding="utf-8") as f:
|
|
91
|
+
lines = f.read().splitlines()
|
|
92
|
+
if lines:
|
|
93
|
+
from .util import atomic_write
|
|
94
|
+
atomic_write(self.path, "\n".join(lines[:-1]) + ("\n" if lines[:-1] else ""))
|