eggdb 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {eggdb-0.1.0 → eggdb-0.2.0}/Cargo.toml +4 -3
  2. {eggdb-0.1.0 → eggdb-0.2.0}/PKG-INFO +13 -4
  3. {eggdb-0.1.0/python → eggdb-0.2.0}/README.md +12 -3
  4. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/Cargo.toml +1 -6
  5. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/dictionary.rs +3 -2
  6. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/mod.rs +8 -0
  7. eggdb-0.2.0/crates/eggdb-core/src/format/overlay.rs +480 -0
  8. eggdb-0.2.0/crates/eggdb-core/src/format/rebuild.rs +152 -0
  9. eggdb-0.2.0/crates/eggdb-core/src/format/wal.rs +597 -0
  10. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/lib.rs +6 -0
  11. eggdb-0.2.0/crates/eggdb-core/tests/proptest_overlay.proptest-regressions +7 -0
  12. eggdb-0.2.0/crates/eggdb-core/tests/proptest_overlay.rs +213 -0
  13. eggdb-0.2.0/crates/eggdb-store/Cargo.toml +16 -0
  14. eggdb-0.2.0/crates/eggdb-store/README.md +48 -0
  15. eggdb-0.2.0/crates/eggdb-store/src/compact.rs +261 -0
  16. eggdb-0.2.0/crates/eggdb-store/src/error.rs +95 -0
  17. eggdb-0.2.0/crates/eggdb-store/src/fsutil.rs +171 -0
  18. eggdb-0.2.0/crates/eggdb-store/src/lib.rs +17 -0
  19. eggdb-0.2.0/crates/eggdb-store/src/snapshot.rs +368 -0
  20. eggdb-0.2.0/crates/eggdb-store/src/writer.rs +278 -0
  21. eggdb-0.2.0/crates/eggdb-store/tests/store.rs +502 -0
  22. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/Cargo.toml +2 -1
  23. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/README.md +39 -5
  24. eggdb-0.2.0/crates/eggdb-vtable/src/args.rs +150 -0
  25. eggdb-0.2.0/crates/eggdb-vtable/src/constraints.rs +442 -0
  26. eggdb-0.2.0/crates/eggdb-vtable/src/lib.rs +64 -0
  27. eggdb-0.2.0/crates/eggdb-vtable/src/shared.rs +388 -0
  28. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/src/vtab.rs +287 -338
  29. eggdb-0.2.0/crates/eggdb-vtable/tests/writes.rs +503 -0
  30. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/Cargo.toml +1 -1
  31. {eggdb-0.1.0 → eggdb-0.2.0}/eggdb/__init__.py +1 -1
  32. {eggdb-0.1.0 → eggdb-0.2.0}/pyproject.toml +1 -1
  33. {eggdb-0.1.0 → eggdb-0.2.0}/python/Cargo.lock +139 -31
  34. {eggdb-0.1.0 → eggdb-0.2.0}/python/Cargo.toml +3 -2
  35. {eggdb-0.1.0 → eggdb-0.2.0/python}/README.md +12 -3
  36. {eggdb-0.1.0 → eggdb-0.2.0}/python/src/lib.rs +30 -56
  37. eggdb-0.2.0/python/uv.lock +8 -0
  38. eggdb-0.1.0/crates/eggdb-vtable/src/lib.rs +0 -50
  39. eggdb-0.1.0/python/eggdb/__init__.py +0 -6
  40. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/README.md +0 -0
  41. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/analyzer.rs +0 -0
  42. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/bool_bitmap.rs +0 -0
  43. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/constant.rs +0 -0
  44. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/delta.rs +0 -0
  45. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/mod.rs +0 -0
  46. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/null_bitmap.rs +0 -0
  47. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/raw.rs +0 -0
  48. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/rle.rs +0 -0
  49. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/albumen.rs +0 -0
  50. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/crypto.rs +0 -0
  51. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/header.rs +0 -0
  52. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/membrane.rs +0 -0
  53. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/reader.rs +0 -0
  54. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/validate.rs +0 -0
  55. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/writer.rs +0 -0
  56. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/value.rs +0 -0
  57. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/tests/format_round_trip.rs +0 -0
  58. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/tests/malformed_decode.rs +0 -0
  59. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/tests/proptest_encoders.rs +0 -0
  60. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/src/column_cache.rs +0 -0
  61. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/src/error.rs +0 -0
  62. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/README.md +0 -0
  63. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/ast.rs +0 -0
  64. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/lib.rs +0 -0
  65. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/parser.rs +0 -0
  66. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/sql_gen.rs +0 -0
  67. {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/tokenizer.rs +0 -0
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "egg"
3
- version = "0.1.0"
3
+ version = "0.2.0"
4
4
  edition = "2021"
5
5
  description = "A lifecycle-aware compressed database engine — converts SQLite to highly compressed .egg files"
6
6
  license = "MIT OR Apache-2.0"
@@ -13,6 +13,7 @@ path = "src/main.rs"
13
13
  [dependencies]
14
14
  eggdb-core = { path = "crates/eggdb-core" }
15
15
  eggdb-io = { path = "crates/eggdb-io" }
16
+ eggdb-store = { path = "crates/eggdb-store" }
16
17
  eggdb-yql = { path = "crates/eggdb-yql" }
17
18
  eggdb-vtable = { path = "crates/eggdb-vtable" }
18
19
  bitvec = { workspace = true }
@@ -36,7 +37,7 @@ serde = { workspace = true }
36
37
  tempfile = "3"
37
38
 
38
39
  [workspace]
39
- members = ["crates/eggdb-core", "crates/eggdb-io", "crates/eggdb-vtable", "crates/eggdb-yql"]
40
+ members = ["crates/eggdb-core", "crates/eggdb-io", "crates/eggdb-store", "crates/eggdb-vtable", "crates/eggdb-yql"]
40
41
  exclude = ["fuzz", "python"]
41
42
  resolver = "2"
42
43
 
@@ -48,7 +49,7 @@ zstd = "0.13"
48
49
  crc32c = "0.6"
49
50
  blake3 = "1"
50
51
  thiserror = "2"
51
- rusqlite = { version = "0.31", features = ["bundled", "blob", "vtab"] }
52
+ rusqlite = { version = "=0.40.2", features = ["bundled", "blob", "vtab"] }
52
53
  criterion = { version = "0.5", features = ["html_reports"] }
53
54
  proptest = "1"
54
55
  clap = { version = "4", features = ["derive"] }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: eggdb
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Programming Language :: Rust
6
6
  Classifier: Programming Language :: Python :: Implementation :: CPython
@@ -32,7 +32,7 @@ This package lets you query them directly — no decompression step needed.
32
32
  ## Install
33
33
 
34
34
  ```bash
35
- pip install eggdb
35
+ uv pip install eggdb
36
36
  ```
37
37
 
38
38
  ## Usage
@@ -73,15 +73,24 @@ egg = eggdb.open("encrypted.egg", passphrase="secret")
73
73
  rows = egg.query("SELECT * FROM users LIMIT 5")
74
74
  ```
75
75
 
76
+ ### Pending edits
77
+
78
+ Edits made with `egg query` / `egg shell` (or any SQLite client using
79
+ `mode=rw`) are stored in a `.egg.wal` file until `egg compact` runs. The Python
80
+ API reads them: `query()` results and `summary()["total_rows"]` include pending
81
+ edits, as of when the file was opened. `columns()` statistics describe the
82
+ compressed main file. The Python API itself is read-only; a write statement in
83
+ `query()` fails with a "read-only" error.
84
+
76
85
  ## Development
77
86
 
78
87
  To build from source and test locally:
79
88
 
80
89
  ```bash
81
90
  cd python
82
- python3 -m venv .venv
91
+ uv venv
83
92
  source .venv/bin/activate
84
- pip install maturin
93
+ uv pip install maturin
85
94
  maturin develop --release
86
95
  ```
87
96
 
@@ -8,7 +8,7 @@ This package lets you query them directly — no decompression step needed.
8
8
  ## Install
9
9
 
10
10
  ```bash
11
- pip install eggdb
11
+ uv pip install eggdb
12
12
  ```
13
13
 
14
14
  ## Usage
@@ -49,15 +49,24 @@ egg = eggdb.open("encrypted.egg", passphrase="secret")
49
49
  rows = egg.query("SELECT * FROM users LIMIT 5")
50
50
  ```
51
51
 
52
+ ### Pending edits
53
+
54
+ Edits made with `egg query` / `egg shell` (or any SQLite client using
55
+ `mode=rw`) are stored in a `.egg.wal` file until `egg compact` runs. The Python
56
+ API reads them: `query()` results and `summary()["total_rows"]` include pending
57
+ edits, as of when the file was opened. `columns()` statistics describe the
58
+ compressed main file. The Python API itself is read-only; a write statement in
59
+ `query()` fails with a "read-only" error.
60
+
52
61
  ## Development
53
62
 
54
63
  To build from source and test locally:
55
64
 
56
65
  ```bash
57
66
  cd python
58
- python3 -m venv .venv
67
+ uv venv
59
68
  source .venv/bin/activate
60
- pip install maturin
69
+ uv pip install maturin
61
70
  maturin develop --release
62
71
  ```
63
72
 
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "eggdb-core"
3
- version = "0.1.0"
3
+ version = "0.2.0"
4
4
  edition = "2021"
5
5
  description = "Core library for the .egg file format — encodings, format reader/writer, validation, encryption"
6
6
  license = "MIT OR Apache-2.0"
@@ -28,8 +28,3 @@ parallel = ["dep:rayon"]
28
28
  [dev-dependencies]
29
29
  proptest = { workspace = true }
30
30
  criterion = { workspace = true }
31
-
32
- [[bench]]
33
- name = "encoding"
34
- harness = false
35
- path = "../../benches/encoding.rs"
@@ -35,8 +35,9 @@ impl ColumnEncoder for DictEncoder {
35
35
  }
36
36
  }
37
37
 
38
- // Sort by frequency descending (most common = index 0 = smallest varint)
39
- freq.sort_by(|a, b| b.1.cmp(&a.1));
38
+ // Sort by frequency descending (most common = index 0 = smallest varint).
39
+ // Stable sort: ties keep first-seen order, so output is deterministic.
40
+ freq.sort_by_key(|entry| std::cmp::Reverse(entry.1));
40
41
 
41
42
  let dictionary: Vec<Value> = freq.into_iter().map(|(v, _)| v).collect();
42
43
 
@@ -2,8 +2,11 @@ pub mod albumen;
2
2
  pub mod crypto;
3
3
  pub mod header;
4
4
  pub mod membrane;
5
+ pub mod overlay;
5
6
  pub mod reader;
7
+ pub mod rebuild;
6
8
  pub mod validate;
9
+ pub mod wal;
7
10
  pub mod writer;
8
11
 
9
12
  use thiserror::Error;
@@ -73,6 +76,11 @@ pub enum FormatError {
73
76
  encoding: &'static str,
74
77
  detail: String,
75
78
  },
79
+
80
+ /// The write log (`.egg.wal`) is malformed, or an operation in it is
81
+ /// inconsistent with the table it targets.
82
+ #[error("write log error: {0}")]
83
+ WriteLog(String),
76
84
  }
77
85
 
78
86
  /// Returns the bincode configuration for format version 1.
@@ -0,0 +1,480 @@
1
+ //! Per-table overlay: replays write-log operations on top of a table's
2
+ //! immutable main-file rows. Pure, no IO.
3
+ //!
4
+ //! # Physical row space
5
+ //!
6
+ //! A table's rows are numbered `[0, base_rows + appended)`. Rows below
7
+ //! `base_rows` come from the main file; rows above come from log inserts, in
8
+ //! commit order. **A rowid is its physical index**, so rowids are stable until
9
+ //! compaction: updates replace values in place, deletes leave a tombstone, and
10
+ //! deleted rowids are never reused.
11
+
12
+ use std::collections::BTreeMap;
13
+
14
+ use bitvec::prelude::*;
15
+
16
+ use crate::format::albumen::TableMeta;
17
+ use crate::format::wal::{WalCommit, WalOp};
18
+ use crate::format::FormatError;
19
+ use crate::value::Value;
20
+
21
+ /// How many changes an overlay holds relative to the main file.
22
+ #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
23
+ pub struct PendingCounts {
24
+ /// Rows added through the log that are still live.
25
+ pub inserted: u64,
26
+ /// Main-file rows whose values were replaced and are still live.
27
+ pub updated: u64,
28
+ /// Main-file rows that were deleted.
29
+ pub deleted: u64,
30
+ }
31
+
32
+ impl PendingCounts {
33
+ /// True when the overlay changes nothing visible.
34
+ #[must_use]
35
+ pub fn is_empty(&self) -> bool {
36
+ self.inserted == 0 && self.updated == 0 && self.deleted == 0
37
+ }
38
+ }
39
+
40
+ /// Changes to one table, layered over its main-file rows.
41
+ #[derive(Clone, Debug, PartialEq, Eq)]
42
+ pub struct TableOverlay {
43
+ base_rows: u64,
44
+ column_count: usize,
45
+ /// Tombstones over main-file rows. Empty until the first delete.
46
+ deleted_base: BitVec,
47
+ deleted_base_count: u64,
48
+ /// Replacement values for live main-file rows.
49
+ updated_base: BTreeMap<u64, Vec<Value>>,
50
+ /// Rows inserted through the log. `None` = inserted, then deleted.
51
+ appended: Vec<Option<Vec<Value>>>,
52
+ appended_live: u64,
53
+ }
54
+
55
+ impl TableOverlay {
56
+ /// An empty overlay over `base_rows` main-file rows with `column_count` columns.
57
+ #[must_use]
58
+ pub fn new(base_rows: u64, column_count: usize) -> Self {
59
+ Self {
60
+ base_rows,
61
+ column_count,
62
+ deleted_base: BitVec::new(),
63
+ deleted_base_count: 0,
64
+ updated_base: BTreeMap::new(),
65
+ appended: Vec::new(),
66
+ appended_live: 0,
67
+ }
68
+ }
69
+
70
+ /// Number of rows in the main file.
71
+ #[must_use]
72
+ pub fn base_rows(&self) -> u64 {
73
+ self.base_rows
74
+ }
75
+
76
+ /// Number of physical rows (main-file rows plus every logged insert).
77
+ #[must_use]
78
+ pub fn physical_rows(&self) -> u64 {
79
+ self.base_rows + self.appended.len() as u64
80
+ }
81
+
82
+ /// Number of rows a query sees.
83
+ #[must_use]
84
+ pub fn live_rows(&self) -> u64 {
85
+ self.base_rows - self.deleted_base_count + self.appended_live
86
+ }
87
+
88
+ /// True when the overlay holds no changes at all.
89
+ #[must_use]
90
+ pub fn is_empty(&self) -> bool {
91
+ self.deleted_base_count == 0 && self.updated_base.is_empty() && self.appended.is_empty()
92
+ }
93
+
94
+ /// Change counts relative to the main file.
95
+ #[must_use]
96
+ pub fn pending_counts(&self) -> PendingCounts {
97
+ PendingCounts {
98
+ inserted: self.appended_live,
99
+ updated: self.updated_base.len() as u64,
100
+ deleted: self.deleted_base_count,
101
+ }
102
+ }
103
+
104
+ /// Whether `rowid` names a live row.
105
+ #[must_use]
106
+ pub fn is_live(&self, rowid: u64) -> bool {
107
+ if rowid < self.base_rows {
108
+ !self.is_base_deleted(rowid)
109
+ } else {
110
+ self.appended_slot(rowid)
111
+ .is_some_and(|slot| self.appended[slot].is_some())
112
+ }
113
+ }
114
+
115
+ /// Append a row. Returns its rowid.
116
+ pub fn insert(&mut self, row: Vec<Value>) -> Result<u64, FormatError> {
117
+ self.check_width(&row)?;
118
+ let rowid = self.physical_rows();
119
+ self.appended.push(Some(row));
120
+ self.appended_live += 1;
121
+ Ok(rowid)
122
+ }
123
+
124
+ /// Replace all values of a live row.
125
+ pub fn update(&mut self, rowid: u64, row: Vec<Value>) -> Result<(), FormatError> {
126
+ self.check_width(&row)?;
127
+ self.check_live(rowid, "update")?;
128
+ if rowid < self.base_rows {
129
+ self.updated_base.insert(rowid, row);
130
+ } else if let Some(slot) = self.appended_slot(rowid) {
131
+ self.appended[slot] = Some(row);
132
+ }
133
+ Ok(())
134
+ }
135
+
136
+ /// Delete a live row.
137
+ pub fn delete(&mut self, rowid: u64) -> Result<(), FormatError> {
138
+ self.check_live(rowid, "delete")?;
139
+ if rowid < self.base_rows {
140
+ if self.deleted_base.is_empty() {
141
+ self.deleted_base = bitvec![0; to_index(self.base_rows)?];
142
+ }
143
+ self.deleted_base.set(to_index(rowid)?, true);
144
+ self.deleted_base_count += 1;
145
+ self.updated_base.remove(&rowid);
146
+ } else if let Some(slot) = self.appended_slot(rowid) {
147
+ self.appended[slot] = None;
148
+ self.appended_live -= 1;
149
+ }
150
+ Ok(())
151
+ }
152
+
153
+ /// One column over the physical row space, given that column's main-file
154
+ /// values. Deleted slots hold `Null`; use [`live_mask`](Self::live_mask)
155
+ /// to skip them.
156
+ pub fn physical_column(&self, col: usize, base: Vec<Value>) -> Result<Vec<Value>, FormatError> {
157
+ let mut out = self.patched_base(col, base)?;
158
+ out.reserve(self.appended.len());
159
+ out.extend(
160
+ self.appended
161
+ .iter()
162
+ .map(|slot| slot.as_ref().map_or(Value::Null, |row| row[col].clone())),
163
+ );
164
+ Ok(out)
165
+ }
166
+
167
+ /// One column containing only live rows, in rowid order.
168
+ pub fn visible_column(&self, col: usize, base: Vec<Value>) -> Result<Vec<Value>, FormatError> {
169
+ let mut out = self.patched_base(col, base)?;
170
+ if self.deleted_base_count > 0 {
171
+ let mut idx = 0usize;
172
+ out.retain(|_| {
173
+ let keep = !self.deleted_base[idx];
174
+ idx += 1;
175
+ keep
176
+ });
177
+ }
178
+ out.extend(self.appended.iter().flatten().map(|row| row[col].clone()));
179
+ Ok(out)
180
+ }
181
+
182
+ /// One bit per physical row: 1 = live.
183
+ #[must_use]
184
+ pub fn live_mask(&self) -> BitVec {
185
+ let mut mask = if self.deleted_base.is_empty() {
186
+ bitvec![1; self.base_rows as usize]
187
+ } else {
188
+ !self.deleted_base.clone()
189
+ };
190
+ mask.extend(self.appended.iter().map(Option::is_some));
191
+ mask
192
+ }
193
+
194
+ /// Turn a filter mask computed on main-file rows into a physical-space mask
195
+ /// with no false negatives: updated rows are included (their new value may
196
+ /// match), deleted rows are excluded, and every live appended row is
197
+ /// included. The caller must re-check the filter on returned rows.
198
+ pub fn widen_base_mask(&self, base_mask: BitVec) -> Result<BitVec, FormatError> {
199
+ if base_mask.len() as u64 != self.base_rows {
200
+ return Err(overlay_err(format!(
201
+ "filter mask has {} bits, table has {} main-file rows",
202
+ base_mask.len(),
203
+ self.base_rows
204
+ )));
205
+ }
206
+ let mut mask = base_mask;
207
+ for &rowid in self.updated_base.keys() {
208
+ mask.set(to_index(rowid)?, true);
209
+ }
210
+ if !self.deleted_base.is_empty() {
211
+ mask &= !self.deleted_base.clone();
212
+ }
213
+ mask.extend(self.appended.iter().map(Option::is_some));
214
+ Ok(mask)
215
+ }
216
+
217
+ fn patched_base(&self, col: usize, mut base: Vec<Value>) -> Result<Vec<Value>, FormatError> {
218
+ if col >= self.column_count {
219
+ return Err(overlay_err(format!(
220
+ "column {col} out of range ({} columns)",
221
+ self.column_count
222
+ )));
223
+ }
224
+ if base.len() as u64 != self.base_rows {
225
+ return Err(overlay_err(format!(
226
+ "column {col} has {} main-file values, expected {}",
227
+ base.len(),
228
+ self.base_rows
229
+ )));
230
+ }
231
+ for (&rowid, row) in &self.updated_base {
232
+ base[to_index(rowid)?] = row[col].clone();
233
+ }
234
+ for idx in self.deleted_base.iter_ones() {
235
+ base[idx] = Value::Null;
236
+ }
237
+ Ok(base)
238
+ }
239
+
240
+ fn is_base_deleted(&self, rowid: u64) -> bool {
241
+ !self.deleted_base.is_empty() && self.deleted_base[rowid as usize]
242
+ }
243
+
244
+ fn appended_slot(&self, rowid: u64) -> Option<usize> {
245
+ let slot = usize::try_from(rowid.checked_sub(self.base_rows)?).ok()?;
246
+ (slot < self.appended.len()).then_some(slot)
247
+ }
248
+
249
+ fn check_width(&self, row: &[Value]) -> Result<(), FormatError> {
250
+ if row.len() == self.column_count {
251
+ Ok(())
252
+ } else {
253
+ Err(overlay_err(format!(
254
+ "row has {} values, table has {} columns",
255
+ row.len(),
256
+ self.column_count
257
+ )))
258
+ }
259
+ }
260
+
261
+ fn check_live(&self, rowid: u64, action: &str) -> Result<(), FormatError> {
262
+ if self.is_live(rowid) {
263
+ Ok(())
264
+ } else {
265
+ Err(overlay_err(format!(
266
+ "cannot {action} rowid {rowid}: no such live row"
267
+ )))
268
+ }
269
+ }
270
+ }
271
+
272
+ /// Apply one log operation to the overlay of the table it targets.
273
+ /// Returns the rowid the operation affected.
274
+ pub fn apply_op(overlays: &mut [TableOverlay], op: &WalOp) -> Result<u64, FormatError> {
275
+ let table = op.table();
276
+ let overlay = overlays
277
+ .get_mut(table as usize)
278
+ .ok_or_else(|| overlay_err(format!("operation targets unknown table index {table}")))?;
279
+ match op {
280
+ WalOp::Insert { row, .. } => overlay.insert(row.clone()),
281
+ WalOp::Update { rowid, row, .. } => overlay.update(*rowid, row.clone()).map(|()| *rowid),
282
+ WalOp::Delete { rowid, .. } => overlay.delete(*rowid).map(|()| *rowid),
283
+ }
284
+ }
285
+
286
+ /// Build one overlay per table by replaying committed transactions in order.
287
+ pub fn replay<'a>(
288
+ tables: &[TableMeta],
289
+ commits: impl IntoIterator<Item = &'a WalCommit>,
290
+ ) -> Result<Vec<TableOverlay>, FormatError> {
291
+ let mut overlays: Vec<TableOverlay> = tables
292
+ .iter()
293
+ .map(|t| TableOverlay::new(t.row_count, t.columns.len()))
294
+ .collect();
295
+ for commit in commits {
296
+ for op in &commit.ops {
297
+ apply_op(&mut overlays, op)
298
+ .map_err(|e| overlay_err(format!("replaying commit {}: {e}", commit.seq)))?;
299
+ }
300
+ }
301
+ Ok(overlays)
302
+ }
303
+
304
+ fn to_index(rowid: u64) -> Result<usize, FormatError> {
305
+ usize::try_from(rowid).map_err(|_| overlay_err(format!("rowid {rowid} exceeds address space")))
306
+ }
307
+
308
+ fn overlay_err(msg: impl Into<String>) -> FormatError {
309
+ FormatError::WriteLog(msg.into())
310
+ }
311
+
312
+ #[cfg(test)]
313
+ mod tests {
314
+ use super::*;
315
+
316
+ fn int(n: i64) -> Value {
317
+ Value::Integer(n)
318
+ }
319
+
320
+ fn row(a: i64, b: &str) -> Vec<Value> {
321
+ vec![int(a), Value::Text(b.into())]
322
+ }
323
+
324
+ /// Three main-file rows: (1,a) (2,b) (3,c).
325
+ fn base_cols() -> (Vec<Value>, Vec<Value>) {
326
+ (
327
+ vec![int(1), int(2), int(3)],
328
+ vec![
329
+ Value::Text("a".into()),
330
+ Value::Text("b".into()),
331
+ Value::Text("c".into()),
332
+ ],
333
+ )
334
+ }
335
+
336
+ #[test]
337
+ fn empty_overlay_passes_base_through() {
338
+ let ov = TableOverlay::new(3, 2);
339
+ let (ids, _) = base_cols();
340
+ assert!(ov.is_empty());
341
+ assert_eq!(ov.live_rows(), 3);
342
+ assert_eq!(ov.visible_column(0, ids.clone()).unwrap(), ids);
343
+ assert_eq!(ov.physical_column(0, ids.clone()).unwrap(), ids);
344
+ assert_eq!(ov.live_mask(), bitvec![1, 1, 1]);
345
+ assert!(ov.pending_counts().is_empty());
346
+ }
347
+
348
+ #[test]
349
+ fn insert_appends_with_next_physical_rowid() {
350
+ let mut ov = TableOverlay::new(3, 2);
351
+ assert_eq!(ov.insert(row(4, "d")).unwrap(), 3);
352
+ assert_eq!(ov.insert(row(5, "e")).unwrap(), 4);
353
+ let (ids, _) = base_cols();
354
+ assert_eq!(
355
+ ov.visible_column(0, ids).unwrap(),
356
+ vec![int(1), int(2), int(3), int(4), int(5)]
357
+ );
358
+ assert_eq!(ov.live_rows(), 5);
359
+ assert_eq!(ov.pending_counts().inserted, 2);
360
+ }
361
+
362
+ #[test]
363
+ fn update_keeps_rowid_and_order() {
364
+ let mut ov = TableOverlay::new(3, 2);
365
+ ov.update(1, row(20, "B")).unwrap();
366
+ let (_, names) = base_cols();
367
+ assert_eq!(
368
+ ov.visible_column(1, names).unwrap(),
369
+ vec![
370
+ Value::Text("a".into()),
371
+ Value::Text("B".into()),
372
+ Value::Text("c".into())
373
+ ]
374
+ );
375
+ assert_eq!(ov.pending_counts().updated, 1);
376
+ }
377
+
378
+ #[test]
379
+ fn delete_tombstones_and_rowids_are_not_reused() {
380
+ let mut ov = TableOverlay::new(3, 2);
381
+ ov.delete(0).unwrap();
382
+ let new_id = ov.insert(row(9, "z")).unwrap();
383
+ ov.delete(new_id).unwrap();
384
+ assert_eq!(
385
+ ov.insert(row(10, "y")).unwrap(),
386
+ 4,
387
+ "rowid 3 must not be reused"
388
+ );
389
+
390
+ let (ids, _) = base_cols();
391
+ assert_eq!(
392
+ ov.visible_column(0, ids.clone()).unwrap(),
393
+ vec![int(2), int(3), int(10)]
394
+ );
395
+ assert_eq!(
396
+ ov.physical_column(0, ids).unwrap(),
397
+ vec![Value::Null, int(2), int(3), Value::Null, int(10)]
398
+ );
399
+ assert_eq!(ov.live_mask(), bitvec![0, 1, 1, 0, 1]);
400
+ assert_eq!(ov.live_rows(), 3);
401
+ assert_eq!(
402
+ ov.pending_counts(),
403
+ PendingCounts {
404
+ inserted: 1,
405
+ updated: 0,
406
+ deleted: 1
407
+ }
408
+ );
409
+ }
410
+
411
+ #[test]
412
+ fn delete_after_update_drops_the_update() {
413
+ let mut ov = TableOverlay::new(3, 2);
414
+ ov.update(2, row(30, "C")).unwrap();
415
+ ov.delete(2).unwrap();
416
+ assert_eq!(ov.pending_counts().updated, 0);
417
+ assert_eq!(ov.pending_counts().deleted, 1);
418
+ }
419
+
420
+ #[test]
421
+ fn invalid_operations_are_errors() {
422
+ let mut ov = TableOverlay::new(3, 2);
423
+ ov.delete(1).unwrap();
424
+ assert!(ov.delete(1).is_err(), "double delete");
425
+ assert!(ov.update(1, row(0, "x")).is_err(), "update deleted");
426
+ assert!(ov.update(3, row(0, "x")).is_err(), "update past end");
427
+ assert!(ov.delete(u64::MAX).is_err(), "delete far past end");
428
+ assert!(ov.insert(vec![int(1)]).is_err(), "too few values");
429
+ assert!(ov.update(0, vec![int(1); 3]).is_err(), "too many values");
430
+ assert!(ov.visible_column(2, vec![]).is_err(), "bad column");
431
+ assert!(ov.visible_column(0, vec![int(1)]).is_err(), "short base");
432
+ }
433
+
434
+ #[test]
435
+ fn widen_mask_has_no_false_negatives() {
436
+ let mut ov = TableOverlay::new(3, 2);
437
+ ov.update(0, row(99, "match")).unwrap();
438
+ ov.delete(1).unwrap();
439
+ ov.insert(row(7, "new")).unwrap();
440
+ // Encoding-level filter on old values matched only row 1 (now deleted).
441
+ let widened = ov.widen_base_mask(bitvec![0, 1, 0]).unwrap();
442
+ assert_eq!(widened, bitvec![1, 0, 0, 1]);
443
+ assert!(ov.widen_base_mask(bitvec![0, 1]).is_err());
444
+ }
445
+
446
+ #[test]
447
+ fn replay_applies_commits_in_order_and_reports_bad_ops() {
448
+ let tables = vec![TableMeta {
449
+ table_name: "t".into(),
450
+ row_count: 3,
451
+ original_ddl: "CREATE TABLE t (a, b)".into(),
452
+ columns: vec![],
453
+ index_definitions: vec![],
454
+ }];
455
+ // Zero declared columns: any row width is wrong, so the error path is
456
+ // exercised by the second commit.
457
+ let ok = WalCommit {
458
+ seq: 0,
459
+ ops: vec![WalOp::Delete { table: 0, rowid: 2 }],
460
+ };
461
+ let overlays = replay(&tables, [&ok]).unwrap();
462
+ assert_eq!(overlays[0].live_rows(), 2);
463
+
464
+ let bad = WalCommit {
465
+ seq: 1,
466
+ ops: vec![WalOp::Insert {
467
+ table: 0,
468
+ row: vec![int(1)],
469
+ }],
470
+ };
471
+ let err = replay(&tables, [&ok, &bad]).unwrap_err().to_string();
472
+ assert!(err.contains("commit 1"), "{err}");
473
+
474
+ let unknown = WalCommit {
475
+ seq: 0,
476
+ ops: vec![WalOp::Delete { table: 5, rowid: 0 }],
477
+ };
478
+ assert!(replay(&tables, [&unknown]).is_err());
479
+ }
480
+ }