ember-custody 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ember_custody-0.1.0/LICENSE +152 -0
- ember_custody-0.1.0/NOTICE +5 -0
- ember_custody-0.1.0/PKG-INFO +219 -0
- ember_custody-0.1.0/README.md +189 -0
- ember_custody-0.1.0/pyproject.toml +70 -0
- ember_custody-0.1.0/setup.cfg +4 -0
- ember_custody-0.1.0/src/ember/__init__.py +93 -0
- ember_custody-0.1.0/src/ember/adapters.py +525 -0
- ember_custody-0.1.0/src/ember/audit.py +319 -0
- ember_custody-0.1.0/src/ember/cli.py +351 -0
- ember_custody-0.1.0/src/ember/crypto.py +94 -0
- ember_custody-0.1.0/src/ember/keystore.py +234 -0
- ember_custody-0.1.0/src/ember/ledger.py +314 -0
- ember_custody-0.1.0/src/ember/merkle.py +256 -0
- ember_custody-0.1.0/src/ember/receipts.py +189 -0
- ember_custody-0.1.0/src/ember/records.py +232 -0
- ember_custody-0.1.0/src/ember/reducer.py +292 -0
- ember_custody-0.1.0/src/ember/signing.py +144 -0
- ember_custody-0.1.0/src/ember/vault.py +820 -0
- ember_custody-0.1.0/src/ember/verify.py +577 -0
- ember_custody-0.1.0/src/ember_custody.egg-info/PKG-INFO +219 -0
- ember_custody-0.1.0/src/ember_custody.egg-info/SOURCES.txt +34 -0
- ember_custody-0.1.0/src/ember_custody.egg-info/dependency_links.txt +1 -0
- ember_custody-0.1.0/src/ember_custody.egg-info/entry_points.txt +3 -0
- ember_custody-0.1.0/src/ember_custody.egg-info/requires.txt +8 -0
- ember_custody-0.1.0/src/ember_custody.egg-info/top_level.txt +1 -0
- ember_custody-0.1.0/tests/test_adapters.py +200 -0
- ember_custody-0.1.0/tests/test_calibration.py +146 -0
- ember_custody-0.1.0/tests/test_cli.py +98 -0
- ember_custody-0.1.0/tests/test_crypto_keystore.py +69 -0
- ember_custody-0.1.0/tests/test_erasure_cascade.py +110 -0
- ember_custody-0.1.0/tests/test_erasure_suite.py +29 -0
- ember_custody-0.1.0/tests/test_merkle.py +127 -0
- ember_custody-0.1.0/tests/test_reducer_ledger.py +117 -0
- ember_custody-0.1.0/tests/test_security_edge_cases.py +417 -0
- ember_custody-0.1.0/tests/test_signing.py +58 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the Work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, derivative works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated into the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes
|
|
85
|
+
direct or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work or Contribution
|
|
87
|
+
shall terminate as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent
|
|
98
|
+
notices stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" file as part of its distribution,
|
|
107
|
+
then any Derivative Works that You distribute must include a
|
|
108
|
+
readable copy of the NOTICE file distributed with this Work.
|
|
109
|
+
|
|
110
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
111
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
112
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
113
|
+
this License, without any additional terms or conditions.
|
|
114
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
115
|
+
the terms of any separate license agreement you may have executed
|
|
116
|
+
with Licensor regarding such Contributions.
|
|
117
|
+
|
|
118
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
119
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
120
|
+
except as required for reasonable and customary display of the
|
|
121
|
+
origin of the Work and reproducing the NOTICE file.
|
|
122
|
+
|
|
123
|
+
7. Disclaimer of Warranty. Unless required by applicable law or agreed to
|
|
124
|
+
in writing, Licensor provides the Work (and each Contributor provides
|
|
125
|
+
its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
126
|
+
CONDITIONS OF ANY KIND, either express or implied, including, without
|
|
127
|
+
limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT,
|
|
128
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, or NON-INFRINGEMENT.
|
|
129
|
+
You are solely responsible for determining the applicability of
|
|
130
|
+
this License and assuming any risks associated with Your use of the Work.
|
|
131
|
+
|
|
132
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
133
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
134
|
+
unless required by applicable law (such as deliberate fraud and
|
|
135
|
+
gross negligence) or agreed to in writing, shall any Contributor be
|
|
136
|
+
liable to You for any damages, including any direct, indirect, special,
|
|
137
|
+
incidental, or consequential damages of any character arising as a
|
|
138
|
+
result of this License or out of the use or inability to use the
|
|
139
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
140
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
141
|
+
other commercial damages or losses), even if such Contributor has
|
|
142
|
+
been advised of the possibility of such damages.
|
|
143
|
+
|
|
144
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
145
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
146
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
147
|
+
or other liability obligations and/or rights consistent with this
|
|
148
|
+
License. However, in accepting such obligations, You may act only
|
|
149
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
150
|
+
of any other Contributor.
|
|
151
|
+
|
|
152
|
+
END OF TERMS AND CONDITIONS
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ember-custody
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Cryptographic custody, redactable Merkle receipts, and verifiable cascade erasure for AI memory and RAG stores.
|
|
5
|
+
Author-email: Sanyam Sood <sasood01@wsc.edu>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/Starksood/ember
|
|
8
|
+
Project-URL: Repository, https://github.com/Starksood/ember
|
|
9
|
+
Project-URL: Issues, https://github.com/Starksood/ember/issues
|
|
10
|
+
Keywords: ai-memory,gdpr,right-to-be-forgotten,crypto-shredding,merkle-tree,provenance,mem0,rag,audit-log
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Security :: Cryptography
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
License-File: NOTICE
|
|
23
|
+
Requires-Dist: cryptography>=42.0
|
|
24
|
+
Provides-Extra: mem0
|
|
25
|
+
Requires-Dist: mem0ai>=0.1.0; extra == "mem0"
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
|
+
Requires-Dist: ruff>=0.6.0; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# Ember (`ember-custody`)
|
|
32
|
+
|
|
33
|
+
> **Proof that survives erasure.** Cryptographic custody, redactable Merkle provenance receipts, and adversarially tested cascade deletion for AI memory and RAG stores.
|
|
34
|
+
|
|
35
|
+
Existing memory engines (`Mem0`, `Letta`, `LangChain`, vector databases) know how to store embeddings and delete by top-level `user_id`. Under GDPR Article 17 ("Right to be Forgotten") and enterprise audit requirements, three failures still happen silently:
|
|
36
|
+
|
|
37
|
+
1. **The Erasure / Receipt Trilemma:** If two subjects appear in the same source transcript, a flat SHA-256 document hash forces an impossible choice: delete Subject A cleanly and break Bystander B's provenance receipt, or preserve B's receipt by leaving A's source file unredacted.
|
|
38
|
+
2. **Derived & Cross-Bucket Leaks:** Calling `delete_all(user_id="priya")` deletes raw memories keyed to `"priya"`, leaving downstream derived team summaries, shared workspace notes, and append-only audit logs surfacing `"Priya"` forever.
|
|
39
|
+
3. **Zombie Re-Ingestion & Fake Metrics:** Sync retries re-ingest erased subjects seconds later; naive deletion metrics pass `0 leaks` trivially if a system wipes bystander data along with the subject.
|
|
40
|
+
|
|
41
|
+
**Ember** wraps your existing memory or vector store (~10 lines of Python) to solve all three with explicit, falsifiable cryptographic mechanisms.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Core Guarantees
|
|
46
|
+
|
|
47
|
+
- **Redactable Salted Merkle Provenance (`ember.merkle`, `ember.receipts`):** Documents split into sentence/line/custom leaves with length-prefixed per-leaf 128-bit random nonces (`0x00 || len32(nonce) || nonce || leaf_bytes`, RFC 6962 domain separation). Erasing Subject A destroys A's plaintext and nonce while keeping A's leaf digest so **Bystander B's inclusion proof still verifies** against the unchanged Merkle root — while offline dictionary guessing of A's redacted leaf fails.
|
|
48
|
+
- **Zero-Plaintext Hash-Chained Ledger + Crypto-Shredding (`ember.ledger`, `ember.crypto`, `ember.keystore`):** Append-only SQLite WAL chain (`synchronous=FULL`) encrypts `subjects`, `text`, `quote`, and caller `metadata` with `AES-256-GCM` bound via AAD to `(tenant, item_id, field, key_ref)`. Shared and `derived_from` records use tracked per-record keys linked to every upstream contributor — erasing any contributor destroys both their subject CEK and every shared/derived DEK before issuing external adapter calls, so from-zero replay produces `[erased]` even across transient store timeouts.
|
|
49
|
+
- **Keyed Tombstones & Fail-Closed Certificates (`ember.reducer`, `ember.signing`):** Vault-keyed HMAC-SHA256 tombstones block re-ingestion (`TombstonedSubjectError`) unless an explicit `reconsent_id` is supplied. `erase()` probes the **raw wrapped store directly** and raises `ErasureIncomplete` without issuing an Ed25519 certificate if anything still surfaces.
|
|
50
|
+
- **Zero-Dependency Offline Verifier (`ember/verify.py`):** Copy this single Python stdlib file anywhere to verify Merkle redactions, bystander receipts, and ledger hash chains without third-party packages (`cryptography` is used only when checking Ed25519 certificate signatures).
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Installation
|
|
55
|
+
|
|
56
|
+
Install directly from GitHub (`0.1.0` PyPI Trusted Publishing pending release):
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install "ember-custody @ git+https://github.com/Starksood/ember.git"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Once published to PyPI:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install ember-custody
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Optional Mem0 extra:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install "ember-custody[mem0] @ git+https://github.com/Starksood/ember.git"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
From a local clone (using [`uv`](https://docs.astral.sh/uv/)):
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
git clone https://github.com/Starksood/ember.git
|
|
78
|
+
cd ember
|
|
79
|
+
uv venv --python 3.13 .venv
|
|
80
|
+
uv pip install --python .venv/bin/python -e ".[dev]"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
See [Solving the Erasure / Receipt Trilemma](docs/TRILEMMA.md) for the cryptographic design and two-direction benchmark deep dive.
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## 60-Second Quickstart
|
|
88
|
+
|
|
89
|
+
### 1. Wrap Mem0 (or any Vector Store) with Cascade Custody
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from mem0 import Memory
|
|
93
|
+
from ember import CustodyVault, Mem0Adapter, TombstonedSubjectError
|
|
94
|
+
|
|
95
|
+
vault = CustodyVault.open(
|
|
96
|
+
"./ember_ledger",
|
|
97
|
+
key_dir="./ember_keys", # keep keys separate from ledger backups!
|
|
98
|
+
adapter=Mem0Adapter(Memory(), default_user_id="team_acme"),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
# Register a multi-party intake document (Merkle-indexed, zero raw text in ledger)
|
|
102
|
+
vault.register_source(
|
|
103
|
+
"intake_notes",
|
|
104
|
+
"Priya Raman works in Portland on triage routing. Dana Kim leads pharmacy inventory.",
|
|
105
|
+
origin="onboarding_transcript",
|
|
106
|
+
origin_kind="conversation",
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
r_priya = vault.remember(
|
|
110
|
+
"Priya Raman works in Portland on triage routing.",
|
|
111
|
+
subjects=["priya"],
|
|
112
|
+
source_id="intake_notes",
|
|
113
|
+
quote="Priya Raman works in Portland on triage routing.",
|
|
114
|
+
metadata={"user_id": "priya"},
|
|
115
|
+
)
|
|
116
|
+
r_dana = vault.remember(
|
|
117
|
+
"Dana Kim leads pharmacy inventory.",
|
|
118
|
+
subjects=["dana"],
|
|
119
|
+
source_id="intake_notes",
|
|
120
|
+
quote="Dana Kim leads pharmacy inventory.",
|
|
121
|
+
metadata={"user_id": "dana"},
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
# Derived team summary sits in a different user_id bucket — naive delete_all(user_id="priya") misses it
|
|
125
|
+
vault.remember(
|
|
126
|
+
"Riverside handoff: triage routing pairs with pharmacy inventory.",
|
|
127
|
+
subjects=["team_acme"],
|
|
128
|
+
derived_from=[r_priya.item.item_id, r_dana.item.item_id],
|
|
129
|
+
metadata={"user_id": "team_acme"},
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# Right to be Forgotten: cascades to Mem0 records, derived summaries, shared leaves, and keys
|
|
133
|
+
cert = vault.erase(
|
|
134
|
+
"priya",
|
|
135
|
+
probes=["Where does Priya work?", "Who handles triage routing?"],
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Dana (innocent bystander) and her redacted source proof survive intact
|
|
139
|
+
assert vault.verify_item_receipt(r_dana.item.item_id) is True
|
|
140
|
+
assert vault.search("Dana pharmacy").abstained is False
|
|
141
|
+
|
|
142
|
+
# Re-ingesting Priya without explicit re-consent fails closed
|
|
143
|
+
try:
|
|
144
|
+
vault.remember("Priya Raman works in Portland.", subjects=["priya"])
|
|
145
|
+
except TombstonedSubjectError:
|
|
146
|
+
print("Blocked zombie re-ingestion")
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Run the self-contained examples without external API keys:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
python examples/wrap_mem0.py
|
|
153
|
+
python examples/redactable_transcript.py
|
|
154
|
+
python examples/offline_verify_bundle.py
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### 2. CLI Workflow & Offline Audit
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
ember --vault .ember --key-dir .keys init
|
|
161
|
+
ember --vault .ember --key-dir .keys add-source doc1 --text "Priya lives in Portland. Marcus lives in Leeds."
|
|
162
|
+
ember --vault .ember --key-dir .keys remember "Priya lives in Portland." --subject priya --source-id doc1 --meta email=priya@example.org
|
|
163
|
+
ember --vault .ember --key-dir .keys remember "Marcus lives in Leeds." --subject marcus --source-id doc1
|
|
164
|
+
ember --vault .ember --key-dir .keys erase priya --probe "Where does Priya live?"
|
|
165
|
+
ember --vault .ember --key-dir .keys replay
|
|
166
|
+
ember --vault .ember --key-dir .keys export --out bundle.json
|
|
167
|
+
ember verify bundle.json --require-signature
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Or copy `src/ember/verify.py` anywhere and run with Python standard library alone:
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
python3 verify.py bundle.json
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## Adversarial Benchmark & Two-Direction Calibration
|
|
179
|
+
|
|
180
|
+
Run the built-in 12-subject × 7-attack-class benchmark (`direct`, `paraphrase`, `inference`, `multi_hop`, `attribute`, `temporal`, `aggregate` — 204 attack probes total):
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
ember benchmark
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Measured output comparing a naive `delete_all(user_id=subject)` baseline (where derived team summaries live under a workspace bucket) against Ember:
|
|
187
|
+
|
|
188
|
+
```text
|
|
189
|
+
EMBER ERASURE ADVERSARIAL SUITE — 12 subjects, 204 probes (7 attack classes)
|
|
190
|
+
==============================================================================
|
|
191
|
+
Priya baseline_leaks=17 ember_leaks= 0 derived=1 bystander_ok=True receipt_ok=True replay_clean=True
|
|
192
|
+
... (12 subjects total)
|
|
193
|
+
==============================================================================
|
|
194
|
+
Baseline naive leaks : 204/204
|
|
195
|
+
Ember leaks : 0/204
|
|
196
|
+
Bystanders survived : True
|
|
197
|
+
Receipts intact : True
|
|
198
|
+
Replays content-free : True
|
|
199
|
+
Ed25519 certs valid : True
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Why bystander checks are co-primary: leak rate alone is monotone in deletion — deleting *everything* scores `0/204` leaks. Both `tests/test_erasure_suite.py` and `tests/test_calibration.py` verify both failure directions: under-deletion triggers `ErasureIncomplete`, and over-deletion fails bystander retrieval survival.
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## Honest Threat Model Limits
|
|
207
|
+
|
|
208
|
+
Ember documents what cryptography cannot invent locally (see [`SECURITY.md`](SECURITY.md) and [`docs/THREAT_MODEL.md`](docs/THREAT_MODEL.md)):
|
|
209
|
+
|
|
210
|
+
1. **Key Separation & Backups:** Crypto-shredding protects live logs and snapshots taken *after* erasure only if pre-erasure backups did not copy unshredded keys alongside `ledger.db`. Configure a separate `key_dir` or implement `ember.keystore.KeyStore` over KMS/HSM (`AWS KMS`, `GCP KMS`, `Vault Transit`).
|
|
211
|
+
2. **External Time Anchoring:** Self-signed local `Ed25519` proofs establish internal ordering and integrity; preventing an operator holding both `ledger.db` and `ed25519.json` from backdating the whole chain requires plugging an RFC 3161 TSA or public transparency log via `ember.signing.TimestampAnchor`.
|
|
212
|
+
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## Origin & License
|
|
216
|
+
|
|
217
|
+
Ember distills the verifiable provenance, salted Merkle redaction, crypto-shredding, and two-direction audit primitives developed in [Fireweed](https://github.com/Starksood) into a standalone Apache-2.0 library focused strictly on cryptographic custody.
|
|
218
|
+
|
|
219
|
+
Licensed under the [Apache License 2.0](LICENSE).
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# Ember (`ember-custody`)
|
|
2
|
+
|
|
3
|
+
> **Proof that survives erasure.** Cryptographic custody, redactable Merkle provenance receipts, and adversarially tested cascade deletion for AI memory and RAG stores.
|
|
4
|
+
|
|
5
|
+
Existing memory engines (`Mem0`, `Letta`, `LangChain`, vector databases) know how to store embeddings and delete by top-level `user_id`. Under GDPR Article 17 ("Right to be Forgotten") and enterprise audit requirements, three failures still happen silently:
|
|
6
|
+
|
|
7
|
+
1. **The Erasure / Receipt Trilemma:** If two subjects appear in the same source transcript, a flat SHA-256 document hash forces an impossible choice: delete Subject A cleanly and break Bystander B's provenance receipt, or preserve B's receipt by leaving A's source file unredacted.
|
|
8
|
+
2. **Derived & Cross-Bucket Leaks:** Calling `delete_all(user_id="priya")` deletes raw memories keyed to `"priya"`, leaving downstream derived team summaries, shared workspace notes, and append-only audit logs surfacing `"Priya"` forever.
|
|
9
|
+
3. **Zombie Re-Ingestion & Fake Metrics:** Sync retries re-ingest erased subjects seconds later; naive deletion metrics pass `0 leaks` trivially if a system wipes bystander data along with the subject.
|
|
10
|
+
|
|
11
|
+
**Ember** wraps your existing memory or vector store (~10 lines of Python) to solve all three with explicit, falsifiable cryptographic mechanisms.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Core Guarantees
|
|
16
|
+
|
|
17
|
+
- **Redactable Salted Merkle Provenance (`ember.merkle`, `ember.receipts`):** Documents split into sentence/line/custom leaves with length-prefixed per-leaf 128-bit random nonces (`0x00 || len32(nonce) || nonce || leaf_bytes`, RFC 6962 domain separation). Erasing Subject A destroys A's plaintext and nonce while keeping A's leaf digest so **Bystander B's inclusion proof still verifies** against the unchanged Merkle root — while offline dictionary guessing of A's redacted leaf fails.
|
|
18
|
+
- **Zero-Plaintext Hash-Chained Ledger + Crypto-Shredding (`ember.ledger`, `ember.crypto`, `ember.keystore`):** Append-only SQLite WAL chain (`synchronous=FULL`) encrypts `subjects`, `text`, `quote`, and caller `metadata` with `AES-256-GCM` bound via AAD to `(tenant, item_id, field, key_ref)`. Shared and `derived_from` records use tracked per-record keys linked to every upstream contributor — erasing any contributor destroys both their subject CEK and every shared/derived DEK before issuing external adapter calls, so from-zero replay produces `[erased]` even across transient store timeouts.
|
|
19
|
+
- **Keyed Tombstones & Fail-Closed Certificates (`ember.reducer`, `ember.signing`):** Vault-keyed HMAC-SHA256 tombstones block re-ingestion (`TombstonedSubjectError`) unless an explicit `reconsent_id` is supplied. `erase()` probes the **raw wrapped store directly** and raises `ErasureIncomplete` without issuing an Ed25519 certificate if anything still surfaces.
|
|
20
|
+
- **Zero-Dependency Offline Verifier (`ember/verify.py`):** Copy this single Python stdlib file anywhere to verify Merkle redactions, bystander receipts, and ledger hash chains without third-party packages (`cryptography` is used only when checking Ed25519 certificate signatures).
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
Install directly from GitHub (`0.1.0` PyPI Trusted Publishing pending release):
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install "ember-custody @ git+https://github.com/Starksood/ember.git"
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Once published to PyPI:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install ember-custody
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Optional Mem0 extra:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install "ember-custody[mem0] @ git+https://github.com/Starksood/ember.git"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
From a local clone (using [`uv`](https://docs.astral.sh/uv/)):
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
git clone https://github.com/Starksood/ember.git
|
|
48
|
+
cd ember
|
|
49
|
+
uv venv --python 3.13 .venv
|
|
50
|
+
uv pip install --python .venv/bin/python -e ".[dev]"
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
See [Solving the Erasure / Receipt Trilemma](docs/TRILEMMA.md) for the cryptographic design and two-direction benchmark deep dive.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## 60-Second Quickstart
|
|
58
|
+
|
|
59
|
+
### 1. Wrap Mem0 (or any Vector Store) with Cascade Custody
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from mem0 import Memory
|
|
63
|
+
from ember import CustodyVault, Mem0Adapter, TombstonedSubjectError
|
|
64
|
+
|
|
65
|
+
vault = CustodyVault.open(
|
|
66
|
+
"./ember_ledger",
|
|
67
|
+
key_dir="./ember_keys", # keep keys separate from ledger backups!
|
|
68
|
+
adapter=Mem0Adapter(Memory(), default_user_id="team_acme"),
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
# Register a multi-party intake document (Merkle-indexed, zero raw text in ledger)
|
|
72
|
+
vault.register_source(
|
|
73
|
+
"intake_notes",
|
|
74
|
+
"Priya Raman works in Portland on triage routing. Dana Kim leads pharmacy inventory.",
|
|
75
|
+
origin="onboarding_transcript",
|
|
76
|
+
origin_kind="conversation",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
r_priya = vault.remember(
|
|
80
|
+
"Priya Raman works in Portland on triage routing.",
|
|
81
|
+
subjects=["priya"],
|
|
82
|
+
source_id="intake_notes",
|
|
83
|
+
quote="Priya Raman works in Portland on triage routing.",
|
|
84
|
+
metadata={"user_id": "priya"},
|
|
85
|
+
)
|
|
86
|
+
r_dana = vault.remember(
|
|
87
|
+
"Dana Kim leads pharmacy inventory.",
|
|
88
|
+
subjects=["dana"],
|
|
89
|
+
source_id="intake_notes",
|
|
90
|
+
quote="Dana Kim leads pharmacy inventory.",
|
|
91
|
+
metadata={"user_id": "dana"},
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# Derived team summary sits in a different user_id bucket — naive delete_all(user_id="priya") misses it
|
|
95
|
+
vault.remember(
|
|
96
|
+
"Riverside handoff: triage routing pairs with pharmacy inventory.",
|
|
97
|
+
subjects=["team_acme"],
|
|
98
|
+
derived_from=[r_priya.item.item_id, r_dana.item.item_id],
|
|
99
|
+
metadata={"user_id": "team_acme"},
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# Right to be Forgotten: cascades to Mem0 records, derived summaries, shared leaves, and keys
|
|
103
|
+
cert = vault.erase(
|
|
104
|
+
"priya",
|
|
105
|
+
probes=["Where does Priya work?", "Who handles triage routing?"],
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# Dana (innocent bystander) and her redacted source proof survive intact
|
|
109
|
+
assert vault.verify_item_receipt(r_dana.item.item_id) is True
|
|
110
|
+
assert vault.search("Dana pharmacy").abstained is False
|
|
111
|
+
|
|
112
|
+
# Re-ingesting Priya without explicit re-consent fails closed
|
|
113
|
+
try:
|
|
114
|
+
vault.remember("Priya Raman works in Portland.", subjects=["priya"])
|
|
115
|
+
except TombstonedSubjectError:
|
|
116
|
+
print("Blocked zombie re-ingestion")
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Run the self-contained examples without external API keys:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
python examples/wrap_mem0.py
|
|
123
|
+
python examples/redactable_transcript.py
|
|
124
|
+
python examples/offline_verify_bundle.py
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### 2. CLI Workflow & Offline Audit
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
ember --vault .ember --key-dir .keys init
|
|
131
|
+
ember --vault .ember --key-dir .keys add-source doc1 --text "Priya lives in Portland. Marcus lives in Leeds."
|
|
132
|
+
ember --vault .ember --key-dir .keys remember "Priya lives in Portland." --subject priya --source-id doc1 --meta email=priya@example.org
|
|
133
|
+
ember --vault .ember --key-dir .keys remember "Marcus lives in Leeds." --subject marcus --source-id doc1
|
|
134
|
+
ember --vault .ember --key-dir .keys erase priya --probe "Where does Priya live?"
|
|
135
|
+
ember --vault .ember --key-dir .keys replay
|
|
136
|
+
ember --vault .ember --key-dir .keys export --out bundle.json
|
|
137
|
+
ember verify bundle.json --require-signature
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Or copy `src/ember/verify.py` anywhere and run with Python standard library alone:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
python3 verify.py bundle.json
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
## Adversarial Benchmark & Two-Direction Calibration
|
|
149
|
+
|
|
150
|
+
Run the built-in 12-subject × 7-attack-class benchmark (`direct`, `paraphrase`, `inference`, `multi_hop`, `attribute`, `temporal`, `aggregate` — 204 attack probes total):
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
ember benchmark
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Measured output comparing a naive `delete_all(user_id=subject)` baseline (where derived team summaries live under a workspace bucket) against Ember:
|
|
157
|
+
|
|
158
|
+
```text
|
|
159
|
+
EMBER ERASURE ADVERSARIAL SUITE — 12 subjects, 204 probes (7 attack classes)
|
|
160
|
+
==============================================================================
|
|
161
|
+
Priya baseline_leaks=17 ember_leaks= 0 derived=1 bystander_ok=True receipt_ok=True replay_clean=True
|
|
162
|
+
... (12 subjects total)
|
|
163
|
+
==============================================================================
|
|
164
|
+
Baseline naive leaks : 204/204
|
|
165
|
+
Ember leaks : 0/204
|
|
166
|
+
Bystanders survived : True
|
|
167
|
+
Receipts intact : True
|
|
168
|
+
Replays content-free : True
|
|
169
|
+
Ed25519 certs valid : True
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Why bystander checks are co-primary: leak rate alone is monotone in deletion — deleting *everything* scores `0/204` leaks. Both `tests/test_erasure_suite.py` and `tests/test_calibration.py` verify both failure directions: under-deletion triggers `ErasureIncomplete`, and over-deletion fails bystander retrieval survival.
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## Honest Threat Model Limits
|
|
177
|
+
|
|
178
|
+
Ember documents what cryptography cannot invent locally (see [`SECURITY.md`](SECURITY.md) and [`docs/THREAT_MODEL.md`](docs/THREAT_MODEL.md)):
|
|
179
|
+
|
|
180
|
+
1. **Key Separation & Backups:** Crypto-shredding protects live logs and snapshots taken *after* erasure only if pre-erasure backups did not copy unshredded keys alongside `ledger.db`. Configure a separate `key_dir` or implement `ember.keystore.KeyStore` over KMS/HSM (`AWS KMS`, `GCP KMS`, `Vault Transit`).
|
|
181
|
+
2. **External Time Anchoring:** Self-signed local `Ed25519` proofs establish internal ordering and integrity; preventing an operator holding both `ledger.db` and `ed25519.json` from backdating the whole chain requires plugging an RFC 3161 TSA or public transparency log via `ember.signing.TimestampAnchor`.
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## Origin & License
|
|
186
|
+
|
|
187
|
+
Ember distills the verifiable provenance, salted Merkle redaction, crypto-shredding, and two-direction audit primitives developed in [Fireweed](https://github.com/Starksood) into a standalone Apache-2.0 library focused strictly on cryptographic custody.
|
|
188
|
+
|
|
189
|
+
Licensed under the [Apache License 2.0](LICENSE).
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ember-custody"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Cryptographic custody, redactable Merkle receipts, and verifiable cascade erasure for AI memory and RAG stores."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "Apache-2.0"
|
|
12
|
+
license-files = ["LICENSE", "NOTICE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Sanyam Sood", email = "sasood01@wsc.edu" },
|
|
15
|
+
]
|
|
16
|
+
keywords = [
|
|
17
|
+
"ai-memory",
|
|
18
|
+
"gdpr",
|
|
19
|
+
"right-to-be-forgotten",
|
|
20
|
+
"crypto-shredding",
|
|
21
|
+
"merkle-tree",
|
|
22
|
+
"provenance",
|
|
23
|
+
"mem0",
|
|
24
|
+
"rag",
|
|
25
|
+
"audit-log",
|
|
26
|
+
]
|
|
27
|
+
classifiers = [
|
|
28
|
+
"Development Status :: 4 - Beta",
|
|
29
|
+
"Intended Audience :: Developers",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Topic :: Security :: Cryptography",
|
|
35
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
36
|
+
]
|
|
37
|
+
dependencies = [
|
|
38
|
+
"cryptography>=42.0",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
mem0 = ["mem0ai>=0.1.0"]
|
|
43
|
+
dev = ["pytest>=8.0", "ruff>=0.6.0"]
|
|
44
|
+
|
|
45
|
+
[project.scripts]
|
|
46
|
+
ember = "ember.cli:main"
|
|
47
|
+
ember-verify = "ember.verify:main"
|
|
48
|
+
|
|
49
|
+
[project.urls]
|
|
50
|
+
Homepage = "https://github.com/Starksood/ember"
|
|
51
|
+
Repository = "https://github.com/Starksood/ember"
|
|
52
|
+
Issues = "https://github.com/Starksood/ember/issues"
|
|
53
|
+
|
|
54
|
+
[tool.setuptools.packages.find]
|
|
55
|
+
where = ["src"]
|
|
56
|
+
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
testpaths = ["tests"]
|
|
59
|
+
addopts = "-q"
|
|
60
|
+
pythonpath = ["src"]
|
|
61
|
+
|
|
62
|
+
[tool.ruff]
|
|
63
|
+
line-length = 120
|
|
64
|
+
target-version = "py311"
|
|
65
|
+
|
|
66
|
+
[tool.ruff.lint]
|
|
67
|
+
select = ["F", "B", "W", "S"]
|
|
68
|
+
|
|
69
|
+
[tool.ruff.lint.per-file-ignores]
|
|
70
|
+
"tests/**" = ["S101", "S603"]
|