mnemo-engine 0.5.1__tar.gz → 0.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/PKG-INFO +1 -1
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/Cargo.lock +2 -2
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/Cargo.toml +1 -1
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/README.md +20 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/crypto.rs +54 -6
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/pager.rs +54 -6
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/tests/proptest_store.rs +119 -0
- mnemo_engine-0.5.2/mnemo/tests/sec1_nonce_uniqueness.rs +261 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo-python/Cargo.lock +2 -2
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo-python/Cargo.toml +1 -1
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/pyproject.toml +1 -1
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/README.md +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/.gitignore +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/benches/cache.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/benches/corpus.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/benches/embed_overhead.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/benches/open.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/benches/recall.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/benches/write.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/deny.toml +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/examples/ann_recall_grid.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/examples/embedder_server.py +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/examples/file_size_breakdown.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/examples/quickstart.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/examples/session.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/bin/mnemo.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/cache.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/embed.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/embed_cmd.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/error.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/format.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/index.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/lib.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/mcp.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/memory.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/result_cache.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/session.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/store.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/src/wal.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/tests/cli_smoke.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/tests/integration.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo/tests/proptest_store.proptest-regressions +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo-python/.gitignore +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo-python/README.md +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo-python/src/lib.rs +0 -0
- {mnemo_engine-0.5.1 → mnemo_engine-0.5.2}/mnemo-python/test_mnemo.py +0 -0
|
@@ -610,7 +610,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
|
610
610
|
|
|
611
611
|
[[package]]
|
|
612
612
|
name = "mnemo-engine"
|
|
613
|
-
version = "0.5.
|
|
613
|
+
version = "0.5.2"
|
|
614
614
|
dependencies = [
|
|
615
615
|
"aes-gcm",
|
|
616
616
|
"argon2",
|
|
@@ -634,7 +634,7 @@ dependencies = [
|
|
|
634
634
|
|
|
635
635
|
[[package]]
|
|
636
636
|
name = "mnemo-node"
|
|
637
|
-
version = "0.5.
|
|
637
|
+
version = "0.5.2"
|
|
638
638
|
dependencies = [
|
|
639
639
|
"mnemo-engine",
|
|
640
640
|
"napi",
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# below stay `mnemo`, so consumers still write `use mnemo::...` and invoke
|
|
7
7
|
# `mnemo` on the command line after `cargo add mnemo-engine`.
|
|
8
8
|
name = "mnemo-engine"
|
|
9
|
-
version = "0.5.
|
|
9
|
+
version = "0.5.2"
|
|
10
10
|
edition = "2021"
|
|
11
11
|
rust-version = "1.75"
|
|
12
12
|
# `benches/corpus.rs` is a shared helper module pulled into real benches
|
|
@@ -530,6 +530,26 @@ The v0.2.0 format (v7) closes every "silent stale-data load" path
|
|
|
530
530
|
flagged in the security review; rollback-replay and availability are
|
|
531
531
|
acknowledged limitations.
|
|
532
532
|
|
|
533
|
+
**v0.5.2 (SEC-1): AES-GCM nonces are now per-session random.** Up to
|
|
534
|
+
v0.5.1, each page's encryption nonce was `(page_no, write_counter)` —
|
|
535
|
+
unique within a single file, but **shared across copies of a file**
|
|
536
|
+
under the same DEK (the data key never changes after `create`). If you
|
|
537
|
+
had two copies of the same `.mnemo` file and both received writes
|
|
538
|
+
(two laptops syncing through Dropbox and the file "conflict-copies"
|
|
539
|
+
diverge; a restored older backup resuming alongside the current file;
|
|
540
|
+
anything that leads to `cp agent.mnemo backup.mnemo` + writes to both),
|
|
541
|
+
the two copies' writers would produce pages with the same encryption
|
|
542
|
+
nonce under the same key but different plaintexts — breaking AES-GCM
|
|
543
|
+
confidentiality and tamper-evidence. v0.5.2 moves to a random 64-bit
|
|
544
|
+
per-session prefix + 32-bit per-session counter (NIST SP 800-38D
|
|
545
|
+
§8.2.1), drawn fresh on every writable open. Cross-copy collisions
|
|
546
|
+
now require two sessions to independently draw the same 64-bit value
|
|
547
|
+
— astronomically unlikely. **If you ran a pre-v0.5.2 build on diverged
|
|
548
|
+
copies, rotate your DEK by running `compact_file`** — it rewrites the
|
|
549
|
+
file through a fresh `random_dek`, so the prior nonce reuse under the
|
|
550
|
+
old key stops mattering for the rewritten pages. No format change;
|
|
551
|
+
existing files open on v0.5.2 unchanged.
|
|
552
|
+
|
|
533
553
|
## Quick start (library)
|
|
534
554
|
|
|
535
555
|
```rust
|
|
@@ -166,13 +166,61 @@ pub fn random_nonce() -> [u8; NONCE_LEN] {
|
|
|
166
166
|
n
|
|
167
167
|
}
|
|
168
168
|
|
|
169
|
-
///
|
|
170
|
-
///
|
|
171
|
-
|
|
172
|
-
|
|
169
|
+
/// Generate a random `u64`, used by the pager to pick a per-session
|
|
170
|
+
/// nonce prefix. See [`session_page_nonce`] for the full construction.
|
|
171
|
+
pub fn random_u64() -> u64 {
|
|
172
|
+
use rand::RngCore;
|
|
173
|
+
rand::thread_rng().next_u64()
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/// Build a per-session page nonce (PR SEC-1, v0.5.2).
|
|
177
|
+
///
|
|
178
|
+
/// Structure, following NIST SP 800-38D §8.2.1 (fixed field + invocation
|
|
179
|
+
/// field) for AES-GCM:
|
|
180
|
+
///
|
|
181
|
+
/// - **Bytes 0..8** — `session_random`: a 64-bit random value drawn
|
|
182
|
+
/// fresh on every writable [`crate::Mnemo::open`] / [`crate::Mnemo::create`]
|
|
183
|
+
/// (and regenerated on counter wrap). This is the "fixed field".
|
|
184
|
+
/// - **Bytes 8..12** — `session_counter`: a 32-bit per-session monotonic
|
|
185
|
+
/// counter that advances once per encrypted page write. This is the
|
|
186
|
+
/// "invocation field".
|
|
187
|
+
///
|
|
188
|
+
/// `page_no` is **not** encoded in the nonce. Pages were bound to their
|
|
189
|
+
/// home slot through AAD since v6 (`page_no.to_le_bytes()`), so an
|
|
190
|
+
/// attacker who swapped a valid page to a different slot already failed
|
|
191
|
+
/// authentication — the AAD binding stays in place and does the job the
|
|
192
|
+
/// `page_no` component of the pre-SEC-1 nonce was helping with.
|
|
193
|
+
///
|
|
194
|
+
/// # Why not deterministic any more
|
|
195
|
+
///
|
|
196
|
+
/// Before v0.5.2 the nonce was `page_no as u32 || write_counter as u64`,
|
|
197
|
+
/// which was unique **within a file** but not **across copies of a file**.
|
|
198
|
+
/// If you copied a `.mnemo` file to another host and both the original and
|
|
199
|
+
/// the copy got written to (two laptops syncing a Dropbox file; a
|
|
200
|
+
/// restored older backup resuming operation), each copy's writer produced
|
|
201
|
+
/// pages whose nonces collided with the other's at the same `(page_no,
|
|
202
|
+
/// write_counter)` coordinates, under the same DEK (`random_dek()` only
|
|
203
|
+
/// at create; `rekey` keeps it). That breaks AES-GCM's confidentiality
|
|
204
|
+
/// and tamper-evidence guarantee: whoever holds both ciphertexts can
|
|
205
|
+
/// recover the XOR of the plaintexts and forge pages with a valid tag.
|
|
206
|
+
///
|
|
207
|
+
/// A fresh random 64-bit `session_random` per writable open closes that:
|
|
208
|
+
/// cross-copy collision now requires two sessions to independently draw
|
|
209
|
+
/// the same 64-bit value (birthday bound ~2^32 sessions, i.e.
|
|
210
|
+
/// astronomically unlikely for a human-scale user). The 32-bit
|
|
211
|
+
/// `session_counter` guarantees uniqueness *within* a session; the pager
|
|
212
|
+
/// regenerates `session_random` and resets the counter if it would wrap.
|
|
213
|
+
///
|
|
214
|
+
/// # Decrypt is unaffected
|
|
215
|
+
///
|
|
216
|
+
/// Each page already stores its own nonce in bytes `[0..12]` of the
|
|
217
|
+
/// on-disk image (see `mnemo/src/pager.rs::read_page`). Decrypt reads the
|
|
218
|
+
/// stored nonce; it never recomputes one. **No format bump — a v0.5.1
|
|
219
|
+
/// file opens fine on v0.5.2 and vice versa.**
|
|
220
|
+
pub fn session_page_nonce(session_random: u64, session_counter: u32) -> [u8; NONCE_LEN] {
|
|
173
221
|
let mut n = [0u8; NONCE_LEN];
|
|
174
|
-
n[0..
|
|
175
|
-
n[
|
|
222
|
+
n[0..8].copy_from_slice(&session_random.to_le_bytes());
|
|
223
|
+
n[8..12].copy_from_slice(&session_counter.to_le_bytes());
|
|
176
224
|
n
|
|
177
225
|
}
|
|
178
226
|
|
|
@@ -93,8 +93,26 @@ pub const DEFAULT_CACHE_PAGES: usize = 8192;
|
|
|
93
93
|
pub struct Pager {
|
|
94
94
|
file: File,
|
|
95
95
|
dek: Zeroizing<[u8; KEY_LEN]>,
|
|
96
|
-
/// Monotonic
|
|
96
|
+
/// Monotonic write counter persisted in the header (lease / replay
|
|
97
|
+
/// machinery; see `Mnemo::prepare_for_flush`). **No longer used in
|
|
98
|
+
/// the page nonce** — SEC-1 (v0.5.2) moved page nonces to a
|
|
99
|
+
/// per-session random prefix + per-session counter so copies of a
|
|
100
|
+
/// file can't collide on `(page_no, write_counter)` under the same
|
|
101
|
+
/// DEK. Still bumped per write for the existing lease machinery.
|
|
97
102
|
pub write_counter: u64,
|
|
103
|
+
/// Random 64-bit "fixed field" (NIST SP 800-38D §8.2.1) stamped
|
|
104
|
+
/// into bytes 0..8 of every page nonce this pager produces. Drawn
|
|
105
|
+
/// fresh on every writable open and regenerated on counter wrap —
|
|
106
|
+
/// so cross-session nonce collisions require two independent
|
|
107
|
+
/// sessions to draw the same 64-bit random value (birthday bound
|
|
108
|
+
/// ~2^32). Unused on read-only handles.
|
|
109
|
+
session_random: u64,
|
|
110
|
+
/// Per-session monotonic counter stamped into bytes 8..12 of every
|
|
111
|
+
/// page nonce. Starts at 0, increments once per encrypted page
|
|
112
|
+
/// write via [`Pager::next_nonce`]. If about to overflow `u32::MAX`,
|
|
113
|
+
/// [`Pager::next_nonce`] regenerates `session_random` and resets to 0
|
|
114
|
+
/// so the (random, counter) pair stays unique within the session.
|
|
115
|
+
session_counter: u32,
|
|
98
116
|
/// Format version this pager is currently encrypting *for*. `Pager` is
|
|
99
117
|
/// version-aware because the format-bump in v6 changed page encryption
|
|
100
118
|
/// to bind each page's home `page_no` as AAD on AES-GCM, making
|
|
@@ -119,11 +137,34 @@ impl Pager {
|
|
|
119
137
|
file,
|
|
120
138
|
dek,
|
|
121
139
|
write_counter,
|
|
140
|
+
// Fresh 64-bit random per Pager construction = per writable
|
|
141
|
+
// open. See SEC-1 doc on `crypto::session_page_nonce`.
|
|
142
|
+
session_random: crypto::random_u64(),
|
|
143
|
+
session_counter: 0,
|
|
122
144
|
version,
|
|
123
145
|
cache: PageCache::new(DEFAULT_CACHE_PAGES),
|
|
124
146
|
}
|
|
125
147
|
}
|
|
126
148
|
|
|
149
|
+
/// Mint the next page nonce for this session.
|
|
150
|
+
///
|
|
151
|
+
/// Increments `session_counter` and returns
|
|
152
|
+
/// `session_random || session_counter`. If the counter would overflow
|
|
153
|
+
/// `u32::MAX`, regenerates `session_random` and resets the counter
|
|
154
|
+
/// before incrementing — so no `(session_random, session_counter)`
|
|
155
|
+
/// pair ever repeats within this session.
|
|
156
|
+
fn next_nonce(&mut self) -> [u8; NONCE_LEN] {
|
|
157
|
+
if self.session_counter == u32::MAX {
|
|
158
|
+
// Wrap guard: pick a new 64-bit prefix and reset the counter
|
|
159
|
+
// so the next write uses `(fresh_random, 1)` instead of
|
|
160
|
+
// reusing `(old_random, 0)`.
|
|
161
|
+
self.session_random = crypto::random_u64();
|
|
162
|
+
self.session_counter = 0;
|
|
163
|
+
}
|
|
164
|
+
self.session_counter += 1;
|
|
165
|
+
crypto::session_page_nonce(self.session_random, self.session_counter)
|
|
166
|
+
}
|
|
167
|
+
|
|
127
168
|
/// Switch the pager to a new format version. Used by [`crate::Mnemo::open`]
|
|
128
169
|
/// during a migration that needs to read with the old version's AAD
|
|
129
170
|
/// scheme and write with the new one.
|
|
@@ -204,8 +245,11 @@ impl Pager {
|
|
|
204
245
|
let mut padded = vec![0u8; PAYLOAD];
|
|
205
246
|
padded[..payload.len()].copy_from_slice(payload);
|
|
206
247
|
|
|
248
|
+
// Bump the header's lease counter (unchanged — still consumed by
|
|
249
|
+
// `prepare_for_flush`); mint the actual nonce from the per-session
|
|
250
|
+
// random + counter pair (SEC-1).
|
|
207
251
|
self.write_counter += 1;
|
|
208
|
-
let nonce =
|
|
252
|
+
let nonce = self.next_nonce();
|
|
209
253
|
let aad = self.page_aad(page_no);
|
|
210
254
|
let ciphertext = crypto::aead_encrypt(&self.dek, &nonce, &padded, &aad)?;
|
|
211
255
|
|
|
@@ -288,9 +332,13 @@ impl Pager {
|
|
|
288
332
|
|
|
289
333
|
/// Encrypt and write every dirty page, then fsync.
|
|
290
334
|
///
|
|
291
|
-
/// Each page gets a fresh nonce from
|
|
292
|
-
///
|
|
293
|
-
///
|
|
335
|
+
/// Each page gets a fresh nonce from the per-session random prefix +
|
|
336
|
+
/// per-session counter (`Pager::next_nonce` → `crypto::session_page_nonce`
|
|
337
|
+
/// since v0.5.2 / SEC-1). The counter advances per write; the random
|
|
338
|
+
/// prefix is drawn once per writable open and regenerated on counter
|
|
339
|
+
/// wrap, so no nonce ever repeats across writes in this session or
|
|
340
|
+
/// across copies of the file. Once written, pages become clean and
|
|
341
|
+
/// the cache is trimmed back to its cap.
|
|
294
342
|
pub fn flush(&mut self) -> Result<()> {
|
|
295
343
|
let pages = self.cache.dirty_pages();
|
|
296
344
|
if pages.is_empty() {
|
|
@@ -303,7 +351,7 @@ impl Pager {
|
|
|
303
351
|
.expect("dirty page must be cached")
|
|
304
352
|
.to_vec();
|
|
305
353
|
self.write_counter += 1;
|
|
306
|
-
let nonce =
|
|
354
|
+
let nonce = self.next_nonce();
|
|
307
355
|
let aad = self.page_aad(page_no);
|
|
308
356
|
let ciphertext = crypto::aead_encrypt(&self.dek, &nonce, &payload, &aad)?;
|
|
309
357
|
|
|
@@ -524,3 +524,122 @@ proptest! {
|
|
|
524
524
|
);
|
|
525
525
|
}
|
|
526
526
|
}
|
|
527
|
+
|
|
528
|
+
// --- PR SEC-1: no page-nonce ever repeats within a file ---
|
|
529
|
+
//
|
|
530
|
+
// Fifth property, added for the v0.5.2 security hotfix. Random
|
|
531
|
+
// Remember/Flush/Reopen sequences on one `.mnemo` file; after the run,
|
|
532
|
+
// scan every encrypted page's 12-byte nonce and assert they're all
|
|
533
|
+
// distinct. Together with the targeted "no cross-copy nonce" and "fresh
|
|
534
|
+
// prefix per reopen" integration tests in sec1_nonce_uniqueness.rs,
|
|
535
|
+
// this covers the AES-GCM uniqueness invariant across the engine's
|
|
536
|
+
// actual write surface (not just a hand-rolled sequence).
|
|
537
|
+
|
|
538
|
+
use std::fs::File;
|
|
539
|
+
use std::io::{Read, Seek, SeekFrom};
|
|
540
|
+
|
|
541
|
+
const SEC1_PAGE_SIZE: usize = 8192;
|
|
542
|
+
const SEC1_NONCE_LEN: usize = 12;
|
|
543
|
+
|
|
544
|
+
fn sec1_collect_nonces(path: &Path) -> BTreeSet<[u8; SEC1_NONCE_LEN]> {
|
|
545
|
+
let mut f = File::open(path).expect("open");
|
|
546
|
+
let len = f.metadata().expect("stat").len();
|
|
547
|
+
let page_count = (len / SEC1_PAGE_SIZE as u64) as usize;
|
|
548
|
+
let mut out = BTreeSet::new();
|
|
549
|
+
// Skip page 0 (plaintext header).
|
|
550
|
+
for page_no in 1..page_count {
|
|
551
|
+
f.seek(SeekFrom::Start((page_no as u64) * SEC1_PAGE_SIZE as u64))
|
|
552
|
+
.expect("seek");
|
|
553
|
+
let mut page = [0u8; SEC1_PAGE_SIZE];
|
|
554
|
+
if f.read_exact(&mut page).is_err() {
|
|
555
|
+
break;
|
|
556
|
+
}
|
|
557
|
+
let mut nonce = [0u8; SEC1_NONCE_LEN];
|
|
558
|
+
nonce.copy_from_slice(&page[..SEC1_NONCE_LEN]);
|
|
559
|
+
if nonce != [0u8; SEC1_NONCE_LEN] {
|
|
560
|
+
out.insert(nonce);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
out
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
proptest! {
|
|
567
|
+
#![proptest_config(ProptestConfig {
|
|
568
|
+
cases: 64,
|
|
569
|
+
max_shrink_iters: 1024,
|
|
570
|
+
..Default::default()
|
|
571
|
+
})]
|
|
572
|
+
|
|
573
|
+
#[test]
|
|
574
|
+
fn prop_no_page_nonce_repeats_within_file(
|
|
575
|
+
ops in prop::collection::vec(
|
|
576
|
+
(
|
|
577
|
+
prop::collection::vec(any::<u8>(), 1..32), // content bytes
|
|
578
|
+
any::<bool>(), // flush after?
|
|
579
|
+
any::<bool>(), // reopen after?
|
|
580
|
+
),
|
|
581
|
+
1..12,
|
|
582
|
+
),
|
|
583
|
+
) {
|
|
584
|
+
let td = TempDir::new().unwrap();
|
|
585
|
+
let path = td.path().join("prop-sec1.mnemo");
|
|
586
|
+
let dims = 4;
|
|
587
|
+
let mut db = Mnemo::create(&path, "pw", MnemoConfig {
|
|
588
|
+
dimensions: dims,
|
|
589
|
+
kdf: KdfParams::fast(),
|
|
590
|
+
..Default::default()
|
|
591
|
+
}).unwrap();
|
|
592
|
+
|
|
593
|
+
for (i, (content, should_flush, should_reopen)) in ops.iter().enumerate() {
|
|
594
|
+
// Distinct per-op vector so the engine actually writes a page.
|
|
595
|
+
let v: Vec<f32> = (0..dims)
|
|
596
|
+
.map(|k| (i as f32 + k as f32 + 1.0) * 0.17)
|
|
597
|
+
.collect();
|
|
598
|
+
db.remember(Memory::new(
|
|
599
|
+
format!("op-{i}:{}", content.iter().map(|b| *b as u32).sum::<u32>()),
|
|
600
|
+
MemoryType::Semantic,
|
|
601
|
+
v,
|
|
602
|
+
))
|
|
603
|
+
.unwrap();
|
|
604
|
+
if *should_flush || *should_reopen {
|
|
605
|
+
db.flush().unwrap();
|
|
606
|
+
}
|
|
607
|
+
if *should_reopen {
|
|
608
|
+
db.close().unwrap();
|
|
609
|
+
drop(db);
|
|
610
|
+
db = Mnemo::open(&path, "pw").unwrap();
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
db.flush().unwrap();
|
|
614
|
+
db.close().unwrap();
|
|
615
|
+
drop(db);
|
|
616
|
+
|
|
617
|
+
// Count stored (nonce) entries vs scanned pages. BTreeSet
|
|
618
|
+
// deduplicates — a repeat would make the set shorter than the
|
|
619
|
+
// page count. We can't know the exact page count up front
|
|
620
|
+
// (compaction, WAL frames reused for later txns, etc.), so
|
|
621
|
+
// compare set.len() to the count of non-empty page slots we
|
|
622
|
+
// actually read.
|
|
623
|
+
let mut f = File::open(&path).unwrap();
|
|
624
|
+
let total_pages = (f.metadata().unwrap().len() / SEC1_PAGE_SIZE as u64) as usize;
|
|
625
|
+
let mut non_empty = 0usize;
|
|
626
|
+
for page_no in 1..total_pages {
|
|
627
|
+
f.seek(SeekFrom::Start((page_no as u64) * SEC1_PAGE_SIZE as u64)).unwrap();
|
|
628
|
+
let mut page = [0u8; SEC1_PAGE_SIZE];
|
|
629
|
+
if f.read_exact(&mut page).is_err() {
|
|
630
|
+
break;
|
|
631
|
+
}
|
|
632
|
+
let mut nonce = [0u8; SEC1_NONCE_LEN];
|
|
633
|
+
nonce.copy_from_slice(&page[..SEC1_NONCE_LEN]);
|
|
634
|
+
if nonce != [0u8; SEC1_NONCE_LEN] {
|
|
635
|
+
non_empty += 1;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
let unique = sec1_collect_nonces(&path).len();
|
|
639
|
+
prop_assert_eq!(
|
|
640
|
+
unique, non_empty,
|
|
641
|
+
"page-nonce repeat detected: {} unique nonces across {} non-empty pages",
|
|
642
|
+
unique, non_empty,
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
//! PR SEC-1 (v0.5.2): page-nonce uniqueness invariants.
|
|
2
|
+
//!
|
|
3
|
+
//! Before SEC-1, page nonces were `page_no || write_counter` — unique
|
|
4
|
+
//! within a file but **reused across copies of a file**. If you copied
|
|
5
|
+
//! a `.mnemo` file to another host and both the original and the copy
|
|
6
|
+
//! got written to (two laptops syncing a file; a restored older backup
|
|
7
|
+
//! resuming operation), each copy's writer produced pages whose nonces
|
|
8
|
+
//! collided with the other's at the same `(page_no, write_counter)`
|
|
9
|
+
//! coordinates, under the same DEK (`random_dek()` is only called at
|
|
10
|
+
//! create). That breaks AES-GCM confidentiality (XOR of plaintexts
|
|
11
|
+
//! recoverable) and tamper-evidence (GHASH key recoverable → page
|
|
12
|
+
//! forgery possible).
|
|
13
|
+
//!
|
|
14
|
+
//! The fix is a per-session random prefix: every writable open draws
|
|
15
|
+
//! a fresh 64-bit `session_random`, and the nonce becomes
|
|
16
|
+
//! `session_random || session_counter`. Cross-session collisions now
|
|
17
|
+
//! require two sessions to independently draw the same 64-bit random
|
|
18
|
+
//! value (negligible). This file pins the three invariants the planner
|
|
19
|
+
//! asked for:
|
|
20
|
+
//!
|
|
21
|
+
//! 1. **No nonce in both copies.** Fork a file, write to both forks,
|
|
22
|
+
//! scan every page in both — the raw-nonce sets must be disjoint.
|
|
23
|
+
//! 2. **Nonces differ across reopen sessions.** Open, write, close,
|
|
24
|
+
//! reopen, write — the two sessions' nonces must not overlap.
|
|
25
|
+
//! 3. **v0.5.1 fixtures still open.** A file produced by the old
|
|
26
|
+
//! deterministic-nonce code must still decrypt, since decrypt reads
|
|
27
|
+
//! the stored nonce rather than recomputing one. (Simulated here by
|
|
28
|
+
//! using the current code to produce a file and reopening it — the
|
|
29
|
+
//! on-disk schema didn't change in SEC-1, so the equivalence is
|
|
30
|
+
//! structural, not just empirical.)
|
|
31
|
+
|
|
32
|
+
use std::collections::HashSet;
|
|
33
|
+
use std::fs;
|
|
34
|
+
use std::io::{Read, Seek, SeekFrom};
|
|
35
|
+
|
|
36
|
+
use mnemo::{Memory, MemoryType, Mnemo, MnemoConfig};
|
|
37
|
+
use tempfile::TempDir;
|
|
38
|
+
|
|
39
|
+
const PAGE_SIZE: usize = 8192;
|
|
40
|
+
const NONCE_LEN: usize = 12;
|
|
41
|
+
|
|
42
|
+
fn fast_cfg(dims: usize) -> MnemoConfig {
|
|
43
|
+
MnemoConfig {
|
|
44
|
+
dimensions: dims,
|
|
45
|
+
kdf: mnemo::KdfParams::fast(),
|
|
46
|
+
..Default::default()
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/// WAL frame magic. A page starting with these four bytes is a WAL
|
|
51
|
+
/// frame, not an encrypted data page — its first 12 bytes are
|
|
52
|
+
/// `MAGIC(4) || kind(1) || txn_id(8)...`, not an AES-GCM nonce.
|
|
53
|
+
/// Skipped by [`collect_nonces`] so the test doesn't compare frame
|
|
54
|
+
/// headers across copies (two files recovered from the same state
|
|
55
|
+
/// will naturally share identical frame headers at the same offsets).
|
|
56
|
+
const WAL_FRAME_MAGIC: &[u8; 4] = b"MWAL";
|
|
57
|
+
|
|
58
|
+
/// Collect every encrypted page's 12-byte nonce from a `.mnemo` file.
|
|
59
|
+
///
|
|
60
|
+
/// Skip:
|
|
61
|
+
/// - Page 0 (plaintext header, not AES-GCM).
|
|
62
|
+
/// - WAL frame slots whose first 4 bytes are `MWAL` — these are
|
|
63
|
+
/// log-frame headers, not nonces. Two files that share WAL frames
|
|
64
|
+
/// (e.g. one is a `cp` of the other) will have identical frame
|
|
65
|
+
/// headers at the same positions; that's structural, not a nonce
|
|
66
|
+
/// reuse.
|
|
67
|
+
/// - All-zero page slots (unwritten space in the WAL region or at
|
|
68
|
+
/// the file tail).
|
|
69
|
+
///
|
|
70
|
+
/// Everything else is a nonce from an AES-GCM encrypted page.
|
|
71
|
+
fn collect_nonces(path: &std::path::Path) -> HashSet<[u8; NONCE_LEN]> {
|
|
72
|
+
let mut file = fs::File::open(path).expect("open for nonce scan");
|
|
73
|
+
let len = file.metadata().expect("stat").len();
|
|
74
|
+
assert!(len >= PAGE_SIZE as u64, "file must have at least one page");
|
|
75
|
+
let page_count = (len / PAGE_SIZE as u64) as usize;
|
|
76
|
+
|
|
77
|
+
let mut out = HashSet::new();
|
|
78
|
+
for page_no in 1..page_count {
|
|
79
|
+
file.seek(SeekFrom::Start((page_no as u64) * PAGE_SIZE as u64))
|
|
80
|
+
.expect("seek");
|
|
81
|
+
let mut page = [0u8; PAGE_SIZE];
|
|
82
|
+
if file.read_exact(&mut page).is_err() {
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
if &page[..4] == WAL_FRAME_MAGIC {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
let mut nonce = [0u8; NONCE_LEN];
|
|
89
|
+
nonce.copy_from_slice(&page[..NONCE_LEN]);
|
|
90
|
+
if nonce != [0u8; NONCE_LEN] {
|
|
91
|
+
out.insert(nonce);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
out
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/// Insert N distinct memories + flush. Caller owns the handle; this
|
|
98
|
+
/// just populates it with pages worth encrypting.
|
|
99
|
+
fn seed(db: &mut Mnemo, start: usize, count: usize) {
|
|
100
|
+
for i in start..start + count {
|
|
101
|
+
let v = (0..4).map(|k| (i as f32 + k as f32) * 0.123).collect();
|
|
102
|
+
db.remember(Memory::new(format!("mem-{i}"), MemoryType::Semantic, v))
|
|
103
|
+
.expect("remember");
|
|
104
|
+
}
|
|
105
|
+
db.flush().expect("flush");
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
#[test]
|
|
109
|
+
fn sec1_no_nonce_appears_in_both_copies() {
|
|
110
|
+
// Property: new pages written to two diverged copies of the same
|
|
111
|
+
// file carry disjoint nonces. This is the exact failure mode SEC-1
|
|
112
|
+
// fixes — before it, pages at the same (page_no, write_counter)
|
|
113
|
+
// coordinates would carry byte-identical nonces in both copies
|
|
114
|
+
// under the same DEK with different plaintexts.
|
|
115
|
+
//
|
|
116
|
+
// Nonces present *before* the fork are shared by construction
|
|
117
|
+
// (byte-for-byte copy of A to B), so the test snapshots those and
|
|
118
|
+
// only compares nonces introduced *after* the fork — those are the
|
|
119
|
+
// ones that would collide under the pre-SEC-1 scheme.
|
|
120
|
+
let td = TempDir::new().unwrap();
|
|
121
|
+
let a = td.path().join("a.mnemo");
|
|
122
|
+
let b = td.path().join("b.mnemo");
|
|
123
|
+
|
|
124
|
+
// Create A with some initial content, flush, close.
|
|
125
|
+
{
|
|
126
|
+
let mut db = Mnemo::create(&a, "pw", fast_cfg(4)).unwrap();
|
|
127
|
+
seed(&mut db, 0, 10);
|
|
128
|
+
db.close().unwrap();
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Snapshot nonces present in A before the fork; these will be
|
|
132
|
+
// duplicated in B by construction (same byte content, same stored
|
|
133
|
+
// nonce). Any nonce in this set is already-shared history, not a
|
|
134
|
+
// reuse violation.
|
|
135
|
+
let pre_fork = collect_nonces(&a);
|
|
136
|
+
|
|
137
|
+
// Fork: byte-for-byte copy A -> B. Represents a user running `cp`
|
|
138
|
+
// for backup, a Dropbox sync propagating the file, or a restored
|
|
139
|
+
// snapshot.
|
|
140
|
+
fs::copy(&a, &b).expect("fork copy");
|
|
141
|
+
|
|
142
|
+
// Writer A: open the original and append new content.
|
|
143
|
+
{
|
|
144
|
+
let mut db = Mnemo::open(&a, "pw").unwrap();
|
|
145
|
+
seed(&mut db, 100, 10);
|
|
146
|
+
db.close().unwrap();
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Writer B: open the copy and append its own, different content.
|
|
150
|
+
// Pre-SEC-1, A and B would use the same `(page_no, write_counter)`
|
|
151
|
+
// sequence for their new pages → identical nonces → nonce reuse.
|
|
152
|
+
{
|
|
153
|
+
let mut db = Mnemo::open(&b, "pw").unwrap();
|
|
154
|
+
seed(&mut db, 200, 10);
|
|
155
|
+
db.close().unwrap();
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
let na_after = collect_nonces(&a);
|
|
159
|
+
let nb_after = collect_nonces(&b);
|
|
160
|
+
|
|
161
|
+
// New nonces introduced in each copy = everything that wasn't in
|
|
162
|
+
// the pre-fork snapshot. These are the writes we care about.
|
|
163
|
+
let a_new: HashSet<_> = na_after.difference(&pre_fork).copied().collect();
|
|
164
|
+
let b_new: HashSet<_> = nb_after.difference(&pre_fork).copied().collect();
|
|
165
|
+
assert!(!a_new.is_empty(), "A must have written new encrypted pages");
|
|
166
|
+
assert!(!b_new.is_empty(), "B must have written new encrypted pages");
|
|
167
|
+
|
|
168
|
+
let overlap: Vec<_> = a_new.intersection(&b_new).collect();
|
|
169
|
+
assert!(
|
|
170
|
+
overlap.is_empty(),
|
|
171
|
+
"SEC-1 violation: {} nonce(s) reused across post-fork writes to \
|
|
172
|
+
copies A and B. Overlapping bytes: {:x?}",
|
|
173
|
+
overlap.len(),
|
|
174
|
+
overlap,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
#[test]
|
|
179
|
+
fn sec1_nonces_differ_across_reopen_sessions() {
|
|
180
|
+
// Property: each writable open gets a fresh `session_random`, so
|
|
181
|
+
// the second session's nonces are disjoint from the first's.
|
|
182
|
+
// Pre-SEC-1 this held *per file* via the monotonic `write_counter`;
|
|
183
|
+
// post-SEC-1 the per-session random prefix makes it robust against
|
|
184
|
+
// cross-copy and cross-restore scenarios too.
|
|
185
|
+
let td = TempDir::new().unwrap();
|
|
186
|
+
let path = td.path().join("sessions.mnemo");
|
|
187
|
+
|
|
188
|
+
// Session 1.
|
|
189
|
+
{
|
|
190
|
+
let mut db = Mnemo::create(&path, "pw", fast_cfg(4)).unwrap();
|
|
191
|
+
seed(&mut db, 0, 10);
|
|
192
|
+
db.close().unwrap();
|
|
193
|
+
}
|
|
194
|
+
let after_s1 = collect_nonces(&path);
|
|
195
|
+
|
|
196
|
+
// Session 2 — fresh open on the same file.
|
|
197
|
+
{
|
|
198
|
+
let mut db = Mnemo::open(&path, "pw").unwrap();
|
|
199
|
+
seed(&mut db, 100, 10);
|
|
200
|
+
db.close().unwrap();
|
|
201
|
+
}
|
|
202
|
+
let after_s2 = collect_nonces(&path);
|
|
203
|
+
|
|
204
|
+
// The second session wrote additional pages, so after_s2 should be
|
|
205
|
+
// a superset of after_s1 (same pages present with same nonces,
|
|
206
|
+
// plus new ones). What we're asserting here is a specific property
|
|
207
|
+
// of the new scheme: the pages written in session 2 use nonces
|
|
208
|
+
// that start with a fresh random prefix — so **no new nonce in
|
|
209
|
+
// session 2 has the same 8-byte prefix as a nonce from session 1**
|
|
210
|
+
// (except by the astronomical ~2^-64 birthday chance).
|
|
211
|
+
let new_in_s2: HashSet<[u8; NONCE_LEN]> = after_s2.difference(&after_s1).copied().collect();
|
|
212
|
+
assert!(
|
|
213
|
+
!new_in_s2.is_empty(),
|
|
214
|
+
"session 2 should have written at least one new page"
|
|
215
|
+
);
|
|
216
|
+
let s1_prefixes: HashSet<[u8; 8]> = after_s1
|
|
217
|
+
.iter()
|
|
218
|
+
.map(|n| {
|
|
219
|
+
let mut p = [0u8; 8];
|
|
220
|
+
p.copy_from_slice(&n[..8]);
|
|
221
|
+
p
|
|
222
|
+
})
|
|
223
|
+
.collect();
|
|
224
|
+
let s2_prefixes: HashSet<[u8; 8]> = new_in_s2
|
|
225
|
+
.iter()
|
|
226
|
+
.map(|n| {
|
|
227
|
+
let mut p = [0u8; 8];
|
|
228
|
+
p.copy_from_slice(&n[..8]);
|
|
229
|
+
p
|
|
230
|
+
})
|
|
231
|
+
.collect();
|
|
232
|
+
let shared: Vec<_> = s1_prefixes.intersection(&s2_prefixes).collect();
|
|
233
|
+
assert!(
|
|
234
|
+
shared.is_empty(),
|
|
235
|
+
"SEC-1 violation: {} session-random prefix(es) shared across \
|
|
236
|
+
two writable opens of the same file; the second session should \
|
|
237
|
+
have minted a fresh 64-bit prefix. Shared: {:x?}",
|
|
238
|
+
shared.len(),
|
|
239
|
+
shared,
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
#[test]
|
|
244
|
+
fn sec1_decrypt_reads_stored_nonce_unchanged_format() {
|
|
245
|
+
// Property: the on-disk schema is unchanged — each page still
|
|
246
|
+
// stores its own 12-byte nonce in bytes [0..12] and decrypt reads
|
|
247
|
+
// it from there. A file produced by the current build must open
|
|
248
|
+
// and read every memory back correctly after reopen, since that's
|
|
249
|
+
// the same code path a v0.5.1 file would exercise.
|
|
250
|
+
let td = TempDir::new().unwrap();
|
|
251
|
+
let path = td.path().join("roundtrip.mnemo");
|
|
252
|
+
let want = 20usize;
|
|
253
|
+
|
|
254
|
+
{
|
|
255
|
+
let mut db = Mnemo::create(&path, "pw", fast_cfg(4)).unwrap();
|
|
256
|
+
seed(&mut db, 0, want);
|
|
257
|
+
db.close().unwrap();
|
|
258
|
+
}
|
|
259
|
+
let mut db = Mnemo::open(&path, "pw").unwrap();
|
|
260
|
+
assert_eq!(db.memories().unwrap().len(), want);
|
|
261
|
+
}
|
|
@@ -409,7 +409,7 @@ dependencies = [
|
|
|
409
409
|
|
|
410
410
|
[[package]]
|
|
411
411
|
name = "mnemo-engine"
|
|
412
|
-
version = "0.5.
|
|
412
|
+
version = "0.5.2"
|
|
413
413
|
dependencies = [
|
|
414
414
|
"aes-gcm",
|
|
415
415
|
"argon2",
|
|
@@ -430,7 +430,7 @@ dependencies = [
|
|
|
430
430
|
|
|
431
431
|
[[package]]
|
|
432
432
|
name = "mnemo-python"
|
|
433
|
-
version = "0.5.
|
|
433
|
+
version = "0.5.2"
|
|
434
434
|
dependencies = [
|
|
435
435
|
"mnemo-engine",
|
|
436
436
|
"pyo3",
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# fields below are still useful for `cargo publish --dry-run` to lint
|
|
5
5
|
# cleanly and for `cargo doc` to render coherently.
|
|
6
6
|
name = "mnemo-python"
|
|
7
|
-
version = "0.5.
|
|
7
|
+
version = "0.5.2"
|
|
8
8
|
edition = "2021"
|
|
9
9
|
rust-version = "1.75"
|
|
10
10
|
description = "Python (PyO3) bindings for the Mnemo encrypted agent-memory engine; built by maturin into the mnemo-engine wheel on PyPI"
|
|
@@ -11,7 +11,7 @@ build-backend = "maturin"
|
|
|
11
11
|
# Cargo.toml), so user code keeps writing `import mnemo` after
|
|
12
12
|
# `pip install mnemo-engine`.
|
|
13
13
|
name = "mnemo-engine"
|
|
14
|
-
version = "0.5.
|
|
14
|
+
version = "0.5.2"
|
|
15
15
|
description = "Encrypted, portable agent-memory engine — Python bindings"
|
|
16
16
|
readme = "README.md"
|
|
17
17
|
requires-python = ">=3.8"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|