eggdb 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eggdb-0.1.0 → eggdb-0.2.0}/Cargo.toml +4 -3
- {eggdb-0.1.0 → eggdb-0.2.0}/PKG-INFO +13 -4
- {eggdb-0.1.0/python → eggdb-0.2.0}/README.md +12 -3
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/Cargo.toml +1 -6
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/dictionary.rs +3 -2
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/mod.rs +8 -0
- eggdb-0.2.0/crates/eggdb-core/src/format/overlay.rs +480 -0
- eggdb-0.2.0/crates/eggdb-core/src/format/rebuild.rs +152 -0
- eggdb-0.2.0/crates/eggdb-core/src/format/wal.rs +597 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/lib.rs +6 -0
- eggdb-0.2.0/crates/eggdb-core/tests/proptest_overlay.proptest-regressions +7 -0
- eggdb-0.2.0/crates/eggdb-core/tests/proptest_overlay.rs +213 -0
- eggdb-0.2.0/crates/eggdb-store/Cargo.toml +16 -0
- eggdb-0.2.0/crates/eggdb-store/README.md +48 -0
- eggdb-0.2.0/crates/eggdb-store/src/compact.rs +261 -0
- eggdb-0.2.0/crates/eggdb-store/src/error.rs +95 -0
- eggdb-0.2.0/crates/eggdb-store/src/fsutil.rs +171 -0
- eggdb-0.2.0/crates/eggdb-store/src/lib.rs +17 -0
- eggdb-0.2.0/crates/eggdb-store/src/snapshot.rs +368 -0
- eggdb-0.2.0/crates/eggdb-store/src/writer.rs +278 -0
- eggdb-0.2.0/crates/eggdb-store/tests/store.rs +502 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/Cargo.toml +2 -1
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/README.md +39 -5
- eggdb-0.2.0/crates/eggdb-vtable/src/args.rs +150 -0
- eggdb-0.2.0/crates/eggdb-vtable/src/constraints.rs +442 -0
- eggdb-0.2.0/crates/eggdb-vtable/src/lib.rs +64 -0
- eggdb-0.2.0/crates/eggdb-vtable/src/shared.rs +388 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/src/vtab.rs +287 -338
- eggdb-0.2.0/crates/eggdb-vtable/tests/writes.rs +503 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/Cargo.toml +1 -1
- {eggdb-0.1.0 → eggdb-0.2.0}/eggdb/__init__.py +1 -1
- {eggdb-0.1.0 → eggdb-0.2.0}/pyproject.toml +1 -1
- {eggdb-0.1.0 → eggdb-0.2.0}/python/Cargo.lock +139 -31
- {eggdb-0.1.0 → eggdb-0.2.0}/python/Cargo.toml +3 -2
- {eggdb-0.1.0 → eggdb-0.2.0/python}/README.md +12 -3
- {eggdb-0.1.0 → eggdb-0.2.0}/python/src/lib.rs +30 -56
- eggdb-0.2.0/python/uv.lock +8 -0
- eggdb-0.1.0/crates/eggdb-vtable/src/lib.rs +0 -50
- eggdb-0.1.0/python/eggdb/__init__.py +0 -6
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/README.md +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/analyzer.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/bool_bitmap.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/constant.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/delta.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/mod.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/null_bitmap.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/raw.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/encoding/rle.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/albumen.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/crypto.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/header.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/membrane.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/reader.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/validate.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/format/writer.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/src/value.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/tests/format_round_trip.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/tests/malformed_decode.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-core/tests/proptest_encoders.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/src/column_cache.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-vtable/src/error.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/README.md +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/ast.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/lib.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/parser.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/sql_gen.rs +0 -0
- {eggdb-0.1.0 → eggdb-0.2.0}/crates/eggdb-yql/src/tokenizer.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "egg"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
description = "A lifecycle-aware compressed database engine — converts SQLite to highly compressed .egg files"
|
|
6
6
|
license = "MIT OR Apache-2.0"
|
|
@@ -13,6 +13,7 @@ path = "src/main.rs"
|
|
|
13
13
|
[dependencies]
|
|
14
14
|
eggdb-core = { path = "crates/eggdb-core" }
|
|
15
15
|
eggdb-io = { path = "crates/eggdb-io" }
|
|
16
|
+
eggdb-store = { path = "crates/eggdb-store" }
|
|
16
17
|
eggdb-yql = { path = "crates/eggdb-yql" }
|
|
17
18
|
eggdb-vtable = { path = "crates/eggdb-vtable" }
|
|
18
19
|
bitvec = { workspace = true }
|
|
@@ -36,7 +37,7 @@ serde = { workspace = true }
|
|
|
36
37
|
tempfile = "3"
|
|
37
38
|
|
|
38
39
|
[workspace]
|
|
39
|
-
members = ["crates/eggdb-core", "crates/eggdb-io", "crates/eggdb-vtable", "crates/eggdb-yql"]
|
|
40
|
+
members = ["crates/eggdb-core", "crates/eggdb-io", "crates/eggdb-store", "crates/eggdb-vtable", "crates/eggdb-yql"]
|
|
40
41
|
exclude = ["fuzz", "python"]
|
|
41
42
|
resolver = "2"
|
|
42
43
|
|
|
@@ -48,7 +49,7 @@ zstd = "0.13"
|
|
|
48
49
|
crc32c = "0.6"
|
|
49
50
|
blake3 = "1"
|
|
50
51
|
thiserror = "2"
|
|
51
|
-
rusqlite = { version = "0.
|
|
52
|
+
rusqlite = { version = "=0.40.2", features = ["bundled", "blob", "vtab"] }
|
|
52
53
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
53
54
|
proptest = "1"
|
|
54
55
|
clap = { version = "4", features = ["derive"] }
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: eggdb
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Programming Language :: Rust
|
|
6
6
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
@@ -32,7 +32,7 @@ This package lets you query them directly — no decompression step needed.
|
|
|
32
32
|
## Install
|
|
33
33
|
|
|
34
34
|
```bash
|
|
35
|
-
pip install eggdb
|
|
35
|
+
uv pip install eggdb
|
|
36
36
|
```
|
|
37
37
|
|
|
38
38
|
## Usage
|
|
@@ -73,15 +73,24 @@ egg = eggdb.open("encrypted.egg", passphrase="secret")
|
|
|
73
73
|
rows = egg.query("SELECT * FROM users LIMIT 5")
|
|
74
74
|
```
|
|
75
75
|
|
|
76
|
+
### Pending edits
|
|
77
|
+
|
|
78
|
+
Edits made with `egg query` / `egg shell` (or any SQLite client using
|
|
79
|
+
`mode=rw`) are stored in a `.egg.wal` file until `egg compact` runs. The Python
|
|
80
|
+
API reads them: `query()` results and `summary()["total_rows"]` include pending
|
|
81
|
+
edits, as of when the file was opened. `columns()` statistics describe the
|
|
82
|
+
compressed main file. The Python API itself is read-only; a write statement in
|
|
83
|
+
`query()` fails with a "read-only" error.
|
|
84
|
+
|
|
76
85
|
## Development
|
|
77
86
|
|
|
78
87
|
To build from source and test locally:
|
|
79
88
|
|
|
80
89
|
```bash
|
|
81
90
|
cd python
|
|
82
|
-
|
|
91
|
+
uv venv
|
|
83
92
|
source .venv/bin/activate
|
|
84
|
-
pip install maturin
|
|
93
|
+
uv pip install maturin
|
|
85
94
|
maturin develop --release
|
|
86
95
|
```
|
|
87
96
|
|
|
@@ -8,7 +8,7 @@ This package lets you query them directly — no decompression step needed.
|
|
|
8
8
|
## Install
|
|
9
9
|
|
|
10
10
|
```bash
|
|
11
|
-
pip install eggdb
|
|
11
|
+
uv pip install eggdb
|
|
12
12
|
```
|
|
13
13
|
|
|
14
14
|
## Usage
|
|
@@ -49,15 +49,24 @@ egg = eggdb.open("encrypted.egg", passphrase="secret")
|
|
|
49
49
|
rows = egg.query("SELECT * FROM users LIMIT 5")
|
|
50
50
|
```
|
|
51
51
|
|
|
52
|
+
### Pending edits
|
|
53
|
+
|
|
54
|
+
Edits made with `egg query` / `egg shell` (or any SQLite client using
|
|
55
|
+
`mode=rw`) are stored in a `.egg.wal` file until `egg compact` runs. The Python
|
|
56
|
+
API reads them: `query()` results and `summary()["total_rows"]` include pending
|
|
57
|
+
edits, as of when the file was opened. `columns()` statistics describe the
|
|
58
|
+
compressed main file. The Python API itself is read-only; a write statement in
|
|
59
|
+
`query()` fails with a "read-only" error.
|
|
60
|
+
|
|
52
61
|
## Development
|
|
53
62
|
|
|
54
63
|
To build from source and test locally:
|
|
55
64
|
|
|
56
65
|
```bash
|
|
57
66
|
cd python
|
|
58
|
-
|
|
67
|
+
uv venv
|
|
59
68
|
source .venv/bin/activate
|
|
60
|
-
pip install maturin
|
|
69
|
+
uv pip install maturin
|
|
61
70
|
maturin develop --release
|
|
62
71
|
```
|
|
63
72
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "eggdb-core"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
description = "Core library for the .egg file format — encodings, format reader/writer, validation, encryption"
|
|
6
6
|
license = "MIT OR Apache-2.0"
|
|
@@ -28,8 +28,3 @@ parallel = ["dep:rayon"]
|
|
|
28
28
|
[dev-dependencies]
|
|
29
29
|
proptest = { workspace = true }
|
|
30
30
|
criterion = { workspace = true }
|
|
31
|
-
|
|
32
|
-
[[bench]]
|
|
33
|
-
name = "encoding"
|
|
34
|
-
harness = false
|
|
35
|
-
path = "../../benches/encoding.rs"
|
|
@@ -35,8 +35,9 @@ impl ColumnEncoder for DictEncoder {
|
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
// Sort by frequency descending (most common = index 0 = smallest varint)
|
|
39
|
-
|
|
38
|
+
// Sort by frequency descending (most common = index 0 = smallest varint).
|
|
39
|
+
// Stable sort: ties keep first-seen order, so output is deterministic.
|
|
40
|
+
freq.sort_by_key(|entry| std::cmp::Reverse(entry.1));
|
|
40
41
|
|
|
41
42
|
let dictionary: Vec<Value> = freq.into_iter().map(|(v, _)| v).collect();
|
|
42
43
|
|
|
@@ -2,8 +2,11 @@ pub mod albumen;
|
|
|
2
2
|
pub mod crypto;
|
|
3
3
|
pub mod header;
|
|
4
4
|
pub mod membrane;
|
|
5
|
+
pub mod overlay;
|
|
5
6
|
pub mod reader;
|
|
7
|
+
pub mod rebuild;
|
|
6
8
|
pub mod validate;
|
|
9
|
+
pub mod wal;
|
|
7
10
|
pub mod writer;
|
|
8
11
|
|
|
9
12
|
use thiserror::Error;
|
|
@@ -73,6 +76,11 @@ pub enum FormatError {
|
|
|
73
76
|
encoding: &'static str,
|
|
74
77
|
detail: String,
|
|
75
78
|
},
|
|
79
|
+
|
|
80
|
+
/// The write log (`.egg.wal`) is malformed, or an operation in it is
|
|
81
|
+
/// inconsistent with the table it targets.
|
|
82
|
+
#[error("write log error: {0}")]
|
|
83
|
+
WriteLog(String),
|
|
76
84
|
}
|
|
77
85
|
|
|
78
86
|
/// Returns the bincode configuration for format version 1.
|
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
//! Per-table overlay: replays write-log operations on top of a table's
|
|
2
|
+
//! immutable main-file rows. Pure, no IO.
|
|
3
|
+
//!
|
|
4
|
+
//! # Physical row space
|
|
5
|
+
//!
|
|
6
|
+
//! A table's rows are numbered `[0, base_rows + appended)`. Rows below
|
|
7
|
+
//! `base_rows` come from the main file; rows above come from log inserts, in
|
|
8
|
+
//! commit order. **A rowid is its physical index**, so rowids are stable until
|
|
9
|
+
//! compaction: updates replace values in place, deletes leave a tombstone, and
|
|
10
|
+
//! deleted rowids are never reused.
|
|
11
|
+
|
|
12
|
+
use std::collections::BTreeMap;
|
|
13
|
+
|
|
14
|
+
use bitvec::prelude::*;
|
|
15
|
+
|
|
16
|
+
use crate::format::albumen::TableMeta;
|
|
17
|
+
use crate::format::wal::{WalCommit, WalOp};
|
|
18
|
+
use crate::format::FormatError;
|
|
19
|
+
use crate::value::Value;
|
|
20
|
+
|
|
21
|
+
/// How many changes an overlay holds relative to the main file.
|
|
22
|
+
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
|
23
|
+
pub struct PendingCounts {
|
|
24
|
+
/// Rows added through the log that are still live.
|
|
25
|
+
pub inserted: u64,
|
|
26
|
+
/// Main-file rows whose values were replaced and are still live.
|
|
27
|
+
pub updated: u64,
|
|
28
|
+
/// Main-file rows that were deleted.
|
|
29
|
+
pub deleted: u64,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
impl PendingCounts {
|
|
33
|
+
/// True when the overlay changes nothing visible.
|
|
34
|
+
#[must_use]
|
|
35
|
+
pub fn is_empty(&self) -> bool {
|
|
36
|
+
self.inserted == 0 && self.updated == 0 && self.deleted == 0
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/// Changes to one table, layered over its main-file rows.
|
|
41
|
+
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
42
|
+
pub struct TableOverlay {
|
|
43
|
+
base_rows: u64,
|
|
44
|
+
column_count: usize,
|
|
45
|
+
/// Tombstones over main-file rows. Empty until the first delete.
|
|
46
|
+
deleted_base: BitVec,
|
|
47
|
+
deleted_base_count: u64,
|
|
48
|
+
/// Replacement values for live main-file rows.
|
|
49
|
+
updated_base: BTreeMap<u64, Vec<Value>>,
|
|
50
|
+
/// Rows inserted through the log. `None` = inserted, then deleted.
|
|
51
|
+
appended: Vec<Option<Vec<Value>>>,
|
|
52
|
+
appended_live: u64,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
impl TableOverlay {
|
|
56
|
+
/// An empty overlay over `base_rows` main-file rows with `column_count` columns.
|
|
57
|
+
#[must_use]
|
|
58
|
+
pub fn new(base_rows: u64, column_count: usize) -> Self {
|
|
59
|
+
Self {
|
|
60
|
+
base_rows,
|
|
61
|
+
column_count,
|
|
62
|
+
deleted_base: BitVec::new(),
|
|
63
|
+
deleted_base_count: 0,
|
|
64
|
+
updated_base: BTreeMap::new(),
|
|
65
|
+
appended: Vec::new(),
|
|
66
|
+
appended_live: 0,
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/// Number of rows in the main file.
|
|
71
|
+
#[must_use]
|
|
72
|
+
pub fn base_rows(&self) -> u64 {
|
|
73
|
+
self.base_rows
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/// Number of physical rows (main-file rows plus every logged insert).
|
|
77
|
+
#[must_use]
|
|
78
|
+
pub fn physical_rows(&self) -> u64 {
|
|
79
|
+
self.base_rows + self.appended.len() as u64
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/// Number of rows a query sees.
|
|
83
|
+
#[must_use]
|
|
84
|
+
pub fn live_rows(&self) -> u64 {
|
|
85
|
+
self.base_rows - self.deleted_base_count + self.appended_live
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/// True when the overlay holds no changes at all.
|
|
89
|
+
#[must_use]
|
|
90
|
+
pub fn is_empty(&self) -> bool {
|
|
91
|
+
self.deleted_base_count == 0 && self.updated_base.is_empty() && self.appended.is_empty()
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/// Change counts relative to the main file.
|
|
95
|
+
#[must_use]
|
|
96
|
+
pub fn pending_counts(&self) -> PendingCounts {
|
|
97
|
+
PendingCounts {
|
|
98
|
+
inserted: self.appended_live,
|
|
99
|
+
updated: self.updated_base.len() as u64,
|
|
100
|
+
deleted: self.deleted_base_count,
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/// Whether `rowid` names a live row.
|
|
105
|
+
#[must_use]
|
|
106
|
+
pub fn is_live(&self, rowid: u64) -> bool {
|
|
107
|
+
if rowid < self.base_rows {
|
|
108
|
+
!self.is_base_deleted(rowid)
|
|
109
|
+
} else {
|
|
110
|
+
self.appended_slot(rowid)
|
|
111
|
+
.is_some_and(|slot| self.appended[slot].is_some())
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/// Append a row. Returns its rowid.
|
|
116
|
+
pub fn insert(&mut self, row: Vec<Value>) -> Result<u64, FormatError> {
|
|
117
|
+
self.check_width(&row)?;
|
|
118
|
+
let rowid = self.physical_rows();
|
|
119
|
+
self.appended.push(Some(row));
|
|
120
|
+
self.appended_live += 1;
|
|
121
|
+
Ok(rowid)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/// Replace all values of a live row.
|
|
125
|
+
pub fn update(&mut self, rowid: u64, row: Vec<Value>) -> Result<(), FormatError> {
|
|
126
|
+
self.check_width(&row)?;
|
|
127
|
+
self.check_live(rowid, "update")?;
|
|
128
|
+
if rowid < self.base_rows {
|
|
129
|
+
self.updated_base.insert(rowid, row);
|
|
130
|
+
} else if let Some(slot) = self.appended_slot(rowid) {
|
|
131
|
+
self.appended[slot] = Some(row);
|
|
132
|
+
}
|
|
133
|
+
Ok(())
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/// Delete a live row.
|
|
137
|
+
pub fn delete(&mut self, rowid: u64) -> Result<(), FormatError> {
|
|
138
|
+
self.check_live(rowid, "delete")?;
|
|
139
|
+
if rowid < self.base_rows {
|
|
140
|
+
if self.deleted_base.is_empty() {
|
|
141
|
+
self.deleted_base = bitvec![0; to_index(self.base_rows)?];
|
|
142
|
+
}
|
|
143
|
+
self.deleted_base.set(to_index(rowid)?, true);
|
|
144
|
+
self.deleted_base_count += 1;
|
|
145
|
+
self.updated_base.remove(&rowid);
|
|
146
|
+
} else if let Some(slot) = self.appended_slot(rowid) {
|
|
147
|
+
self.appended[slot] = None;
|
|
148
|
+
self.appended_live -= 1;
|
|
149
|
+
}
|
|
150
|
+
Ok(())
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/// One column over the physical row space, given that column's main-file
|
|
154
|
+
/// values. Deleted slots hold `Null`; use [`live_mask`](Self::live_mask)
|
|
155
|
+
/// to skip them.
|
|
156
|
+
pub fn physical_column(&self, col: usize, base: Vec<Value>) -> Result<Vec<Value>, FormatError> {
|
|
157
|
+
let mut out = self.patched_base(col, base)?;
|
|
158
|
+
out.reserve(self.appended.len());
|
|
159
|
+
out.extend(
|
|
160
|
+
self.appended
|
|
161
|
+
.iter()
|
|
162
|
+
.map(|slot| slot.as_ref().map_or(Value::Null, |row| row[col].clone())),
|
|
163
|
+
);
|
|
164
|
+
Ok(out)
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/// One column containing only live rows, in rowid order.
|
|
168
|
+
pub fn visible_column(&self, col: usize, base: Vec<Value>) -> Result<Vec<Value>, FormatError> {
|
|
169
|
+
let mut out = self.patched_base(col, base)?;
|
|
170
|
+
if self.deleted_base_count > 0 {
|
|
171
|
+
let mut idx = 0usize;
|
|
172
|
+
out.retain(|_| {
|
|
173
|
+
let keep = !self.deleted_base[idx];
|
|
174
|
+
idx += 1;
|
|
175
|
+
keep
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
out.extend(self.appended.iter().flatten().map(|row| row[col].clone()));
|
|
179
|
+
Ok(out)
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/// One bit per physical row: 1 = live.
|
|
183
|
+
#[must_use]
|
|
184
|
+
pub fn live_mask(&self) -> BitVec {
|
|
185
|
+
let mut mask = if self.deleted_base.is_empty() {
|
|
186
|
+
bitvec![1; self.base_rows as usize]
|
|
187
|
+
} else {
|
|
188
|
+
!self.deleted_base.clone()
|
|
189
|
+
};
|
|
190
|
+
mask.extend(self.appended.iter().map(Option::is_some));
|
|
191
|
+
mask
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/// Turn a filter mask computed on main-file rows into a physical-space mask
|
|
195
|
+
/// with no false negatives: updated rows are included (their new value may
|
|
196
|
+
/// match), deleted rows are excluded, and every live appended row is
|
|
197
|
+
/// included. The caller must re-check the filter on returned rows.
|
|
198
|
+
pub fn widen_base_mask(&self, base_mask: BitVec) -> Result<BitVec, FormatError> {
|
|
199
|
+
if base_mask.len() as u64 != self.base_rows {
|
|
200
|
+
return Err(overlay_err(format!(
|
|
201
|
+
"filter mask has {} bits, table has {} main-file rows",
|
|
202
|
+
base_mask.len(),
|
|
203
|
+
self.base_rows
|
|
204
|
+
)));
|
|
205
|
+
}
|
|
206
|
+
let mut mask = base_mask;
|
|
207
|
+
for &rowid in self.updated_base.keys() {
|
|
208
|
+
mask.set(to_index(rowid)?, true);
|
|
209
|
+
}
|
|
210
|
+
if !self.deleted_base.is_empty() {
|
|
211
|
+
mask &= !self.deleted_base.clone();
|
|
212
|
+
}
|
|
213
|
+
mask.extend(self.appended.iter().map(Option::is_some));
|
|
214
|
+
Ok(mask)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
fn patched_base(&self, col: usize, mut base: Vec<Value>) -> Result<Vec<Value>, FormatError> {
|
|
218
|
+
if col >= self.column_count {
|
|
219
|
+
return Err(overlay_err(format!(
|
|
220
|
+
"column {col} out of range ({} columns)",
|
|
221
|
+
self.column_count
|
|
222
|
+
)));
|
|
223
|
+
}
|
|
224
|
+
if base.len() as u64 != self.base_rows {
|
|
225
|
+
return Err(overlay_err(format!(
|
|
226
|
+
"column {col} has {} main-file values, expected {}",
|
|
227
|
+
base.len(),
|
|
228
|
+
self.base_rows
|
|
229
|
+
)));
|
|
230
|
+
}
|
|
231
|
+
for (&rowid, row) in &self.updated_base {
|
|
232
|
+
base[to_index(rowid)?] = row[col].clone();
|
|
233
|
+
}
|
|
234
|
+
for idx in self.deleted_base.iter_ones() {
|
|
235
|
+
base[idx] = Value::Null;
|
|
236
|
+
}
|
|
237
|
+
Ok(base)
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
fn is_base_deleted(&self, rowid: u64) -> bool {
|
|
241
|
+
!self.deleted_base.is_empty() && self.deleted_base[rowid as usize]
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
fn appended_slot(&self, rowid: u64) -> Option<usize> {
|
|
245
|
+
let slot = usize::try_from(rowid.checked_sub(self.base_rows)?).ok()?;
|
|
246
|
+
(slot < self.appended.len()).then_some(slot)
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
fn check_width(&self, row: &[Value]) -> Result<(), FormatError> {
|
|
250
|
+
if row.len() == self.column_count {
|
|
251
|
+
Ok(())
|
|
252
|
+
} else {
|
|
253
|
+
Err(overlay_err(format!(
|
|
254
|
+
"row has {} values, table has {} columns",
|
|
255
|
+
row.len(),
|
|
256
|
+
self.column_count
|
|
257
|
+
)))
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
fn check_live(&self, rowid: u64, action: &str) -> Result<(), FormatError> {
|
|
262
|
+
if self.is_live(rowid) {
|
|
263
|
+
Ok(())
|
|
264
|
+
} else {
|
|
265
|
+
Err(overlay_err(format!(
|
|
266
|
+
"cannot {action} rowid {rowid}: no such live row"
|
|
267
|
+
)))
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/// Apply one log operation to the overlay of the table it targets.
|
|
273
|
+
/// Returns the rowid the operation affected.
|
|
274
|
+
pub fn apply_op(overlays: &mut [TableOverlay], op: &WalOp) -> Result<u64, FormatError> {
|
|
275
|
+
let table = op.table();
|
|
276
|
+
let overlay = overlays
|
|
277
|
+
.get_mut(table as usize)
|
|
278
|
+
.ok_or_else(|| overlay_err(format!("operation targets unknown table index {table}")))?;
|
|
279
|
+
match op {
|
|
280
|
+
WalOp::Insert { row, .. } => overlay.insert(row.clone()),
|
|
281
|
+
WalOp::Update { rowid, row, .. } => overlay.update(*rowid, row.clone()).map(|()| *rowid),
|
|
282
|
+
WalOp::Delete { rowid, .. } => overlay.delete(*rowid).map(|()| *rowid),
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/// Build one overlay per table by replaying committed transactions in order.
|
|
287
|
+
pub fn replay<'a>(
|
|
288
|
+
tables: &[TableMeta],
|
|
289
|
+
commits: impl IntoIterator<Item = &'a WalCommit>,
|
|
290
|
+
) -> Result<Vec<TableOverlay>, FormatError> {
|
|
291
|
+
let mut overlays: Vec<TableOverlay> = tables
|
|
292
|
+
.iter()
|
|
293
|
+
.map(|t| TableOverlay::new(t.row_count, t.columns.len()))
|
|
294
|
+
.collect();
|
|
295
|
+
for commit in commits {
|
|
296
|
+
for op in &commit.ops {
|
|
297
|
+
apply_op(&mut overlays, op)
|
|
298
|
+
.map_err(|e| overlay_err(format!("replaying commit {}: {e}", commit.seq)))?;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
Ok(overlays)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
fn to_index(rowid: u64) -> Result<usize, FormatError> {
|
|
305
|
+
usize::try_from(rowid).map_err(|_| overlay_err(format!("rowid {rowid} exceeds address space")))
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
fn overlay_err(msg: impl Into<String>) -> FormatError {
|
|
309
|
+
FormatError::WriteLog(msg.into())
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
#[cfg(test)]
|
|
313
|
+
mod tests {
|
|
314
|
+
use super::*;
|
|
315
|
+
|
|
316
|
+
fn int(n: i64) -> Value {
|
|
317
|
+
Value::Integer(n)
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
fn row(a: i64, b: &str) -> Vec<Value> {
|
|
321
|
+
vec![int(a), Value::Text(b.into())]
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/// Three main-file rows: (1,a) (2,b) (3,c).
|
|
325
|
+
fn base_cols() -> (Vec<Value>, Vec<Value>) {
|
|
326
|
+
(
|
|
327
|
+
vec![int(1), int(2), int(3)],
|
|
328
|
+
vec![
|
|
329
|
+
Value::Text("a".into()),
|
|
330
|
+
Value::Text("b".into()),
|
|
331
|
+
Value::Text("c".into()),
|
|
332
|
+
],
|
|
333
|
+
)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
#[test]
|
|
337
|
+
fn empty_overlay_passes_base_through() {
|
|
338
|
+
let ov = TableOverlay::new(3, 2);
|
|
339
|
+
let (ids, _) = base_cols();
|
|
340
|
+
assert!(ov.is_empty());
|
|
341
|
+
assert_eq!(ov.live_rows(), 3);
|
|
342
|
+
assert_eq!(ov.visible_column(0, ids.clone()).unwrap(), ids);
|
|
343
|
+
assert_eq!(ov.physical_column(0, ids.clone()).unwrap(), ids);
|
|
344
|
+
assert_eq!(ov.live_mask(), bitvec![1, 1, 1]);
|
|
345
|
+
assert!(ov.pending_counts().is_empty());
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
#[test]
|
|
349
|
+
fn insert_appends_with_next_physical_rowid() {
|
|
350
|
+
let mut ov = TableOverlay::new(3, 2);
|
|
351
|
+
assert_eq!(ov.insert(row(4, "d")).unwrap(), 3);
|
|
352
|
+
assert_eq!(ov.insert(row(5, "e")).unwrap(), 4);
|
|
353
|
+
let (ids, _) = base_cols();
|
|
354
|
+
assert_eq!(
|
|
355
|
+
ov.visible_column(0, ids).unwrap(),
|
|
356
|
+
vec![int(1), int(2), int(3), int(4), int(5)]
|
|
357
|
+
);
|
|
358
|
+
assert_eq!(ov.live_rows(), 5);
|
|
359
|
+
assert_eq!(ov.pending_counts().inserted, 2);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
#[test]
|
|
363
|
+
fn update_keeps_rowid_and_order() {
|
|
364
|
+
let mut ov = TableOverlay::new(3, 2);
|
|
365
|
+
ov.update(1, row(20, "B")).unwrap();
|
|
366
|
+
let (_, names) = base_cols();
|
|
367
|
+
assert_eq!(
|
|
368
|
+
ov.visible_column(1, names).unwrap(),
|
|
369
|
+
vec![
|
|
370
|
+
Value::Text("a".into()),
|
|
371
|
+
Value::Text("B".into()),
|
|
372
|
+
Value::Text("c".into())
|
|
373
|
+
]
|
|
374
|
+
);
|
|
375
|
+
assert_eq!(ov.pending_counts().updated, 1);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
#[test]
|
|
379
|
+
fn delete_tombstones_and_rowids_are_not_reused() {
|
|
380
|
+
let mut ov = TableOverlay::new(3, 2);
|
|
381
|
+
ov.delete(0).unwrap();
|
|
382
|
+
let new_id = ov.insert(row(9, "z")).unwrap();
|
|
383
|
+
ov.delete(new_id).unwrap();
|
|
384
|
+
assert_eq!(
|
|
385
|
+
ov.insert(row(10, "y")).unwrap(),
|
|
386
|
+
4,
|
|
387
|
+
"rowid 3 must not be reused"
|
|
388
|
+
);
|
|
389
|
+
|
|
390
|
+
let (ids, _) = base_cols();
|
|
391
|
+
assert_eq!(
|
|
392
|
+
ov.visible_column(0, ids.clone()).unwrap(),
|
|
393
|
+
vec![int(2), int(3), int(10)]
|
|
394
|
+
);
|
|
395
|
+
assert_eq!(
|
|
396
|
+
ov.physical_column(0, ids).unwrap(),
|
|
397
|
+
vec![Value::Null, int(2), int(3), Value::Null, int(10)]
|
|
398
|
+
);
|
|
399
|
+
assert_eq!(ov.live_mask(), bitvec![0, 1, 1, 0, 1]);
|
|
400
|
+
assert_eq!(ov.live_rows(), 3);
|
|
401
|
+
assert_eq!(
|
|
402
|
+
ov.pending_counts(),
|
|
403
|
+
PendingCounts {
|
|
404
|
+
inserted: 1,
|
|
405
|
+
updated: 0,
|
|
406
|
+
deleted: 1
|
|
407
|
+
}
|
|
408
|
+
);
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
#[test]
|
|
412
|
+
fn delete_after_update_drops_the_update() {
|
|
413
|
+
let mut ov = TableOverlay::new(3, 2);
|
|
414
|
+
ov.update(2, row(30, "C")).unwrap();
|
|
415
|
+
ov.delete(2).unwrap();
|
|
416
|
+
assert_eq!(ov.pending_counts().updated, 0);
|
|
417
|
+
assert_eq!(ov.pending_counts().deleted, 1);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
#[test]
|
|
421
|
+
fn invalid_operations_are_errors() {
|
|
422
|
+
let mut ov = TableOverlay::new(3, 2);
|
|
423
|
+
ov.delete(1).unwrap();
|
|
424
|
+
assert!(ov.delete(1).is_err(), "double delete");
|
|
425
|
+
assert!(ov.update(1, row(0, "x")).is_err(), "update deleted");
|
|
426
|
+
assert!(ov.update(3, row(0, "x")).is_err(), "update past end");
|
|
427
|
+
assert!(ov.delete(u64::MAX).is_err(), "delete far past end");
|
|
428
|
+
assert!(ov.insert(vec![int(1)]).is_err(), "too few values");
|
|
429
|
+
assert!(ov.update(0, vec![int(1); 3]).is_err(), "too many values");
|
|
430
|
+
assert!(ov.visible_column(2, vec![]).is_err(), "bad column");
|
|
431
|
+
assert!(ov.visible_column(0, vec![int(1)]).is_err(), "short base");
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
#[test]
|
|
435
|
+
fn widen_mask_has_no_false_negatives() {
|
|
436
|
+
let mut ov = TableOverlay::new(3, 2);
|
|
437
|
+
ov.update(0, row(99, "match")).unwrap();
|
|
438
|
+
ov.delete(1).unwrap();
|
|
439
|
+
ov.insert(row(7, "new")).unwrap();
|
|
440
|
+
// Encoding-level filter on old values matched only row 1 (now deleted).
|
|
441
|
+
let widened = ov.widen_base_mask(bitvec![0, 1, 0]).unwrap();
|
|
442
|
+
assert_eq!(widened, bitvec![1, 0, 0, 1]);
|
|
443
|
+
assert!(ov.widen_base_mask(bitvec![0, 1]).is_err());
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
#[test]
|
|
447
|
+
fn replay_applies_commits_in_order_and_reports_bad_ops() {
|
|
448
|
+
let tables = vec![TableMeta {
|
|
449
|
+
table_name: "t".into(),
|
|
450
|
+
row_count: 3,
|
|
451
|
+
original_ddl: "CREATE TABLE t (a, b)".into(),
|
|
452
|
+
columns: vec![],
|
|
453
|
+
index_definitions: vec![],
|
|
454
|
+
}];
|
|
455
|
+
// Zero declared columns: any row width is wrong, so the error path is
|
|
456
|
+
// exercised by the second commit.
|
|
457
|
+
let ok = WalCommit {
|
|
458
|
+
seq: 0,
|
|
459
|
+
ops: vec![WalOp::Delete { table: 0, rowid: 2 }],
|
|
460
|
+
};
|
|
461
|
+
let overlays = replay(&tables, [&ok]).unwrap();
|
|
462
|
+
assert_eq!(overlays[0].live_rows(), 2);
|
|
463
|
+
|
|
464
|
+
let bad = WalCommit {
|
|
465
|
+
seq: 1,
|
|
466
|
+
ops: vec![WalOp::Insert {
|
|
467
|
+
table: 0,
|
|
468
|
+
row: vec![int(1)],
|
|
469
|
+
}],
|
|
470
|
+
};
|
|
471
|
+
let err = replay(&tables, [&ok, &bad]).unwrap_err().to_string();
|
|
472
|
+
assert!(err.contains("commit 1"), "{err}");
|
|
473
|
+
|
|
474
|
+
let unknown = WalCommit {
|
|
475
|
+
seq: 0,
|
|
476
|
+
ops: vec![WalOp::Delete { table: 5, rowid: 0 }],
|
|
477
|
+
};
|
|
478
|
+
assert!(replay(&tables, [&unknown]).is_err());
|
|
479
|
+
}
|
|
480
|
+
}
|