salamander-db 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- salamander_db-0.1.0/Cargo.toml +12 -0
- salamander_db-0.1.0/PKG-INFO +94 -0
- salamander_db-0.1.0/README.md +80 -0
- salamander_db-0.1.0/pyproject.toml +29 -0
- salamander_db-0.1.0/salamander/Cargo.toml +52 -0
- salamander_db-0.1.0/salamander/LICENSE +21 -0
- salamander_db-0.1.0/salamander/README.md +55 -0
- salamander_db-0.1.0/salamander/benches/instant_recovery.rs +85 -0
- salamander_db-0.1.0/salamander/benches/open_time.rs +96 -0
- salamander_db-0.1.0/salamander/benches/replay.rs +141 -0
- salamander_db-0.1.0/salamander/benches/snapshots.rs +69 -0
- salamander_db-0.1.0/salamander/benches/view_apply.rs +82 -0
- salamander_db-0.1.0/salamander/examples/01_kv_basics.rs +65 -0
- salamander_db-0.1.0/salamander/examples/02_custom_payload.rs +70 -0
- salamander_db-0.1.0/salamander/examples/03_query_views.rs +67 -0
- salamander_db-0.1.0/salamander/examples/04_fork.rs +85 -0
- salamander_db-0.1.0/salamander/examples/05_json_and_policy.rs +47 -0
- salamander_db-0.1.0/salamander/examples/06_commit_feed.rs +39 -0
- salamander_db-0.1.0/salamander/examples/07_replicate.rs +18 -0
- salamander_db-0.1.0/salamander/src/agent/kv.rs +48 -0
- salamander_db-0.1.0/salamander/src/agent/mod.rs +150 -0
- salamander_db-0.1.0/salamander/src/agent/session.rs +212 -0
- salamander_db-0.1.0/salamander/src/bin/salamander.rs +54 -0
- salamander_db-0.1.0/salamander/src/branch.rs +248 -0
- salamander_db-0.1.0/salamander/src/commit.rs +149 -0
- salamander_db-0.1.0/salamander/src/db.rs +868 -0
- salamander_db-0.1.0/salamander/src/error.rs +96 -0
- salamander_db-0.1.0/salamander/src/event.rs +40 -0
- salamander_db-0.1.0/salamander/src/facade.rs +3012 -0
- salamander_db-0.1.0/salamander/src/format/mod.rs +878 -0
- salamander_db-0.1.0/salamander/src/introspect.rs +35 -0
- salamander_db-0.1.0/salamander/src/json.rs +113 -0
- salamander_db-0.1.0/salamander/src/lib.rs +121 -0
- salamander_db-0.1.0/salamander/src/log/index.rs +301 -0
- salamander_db-0.1.0/salamander/src/log/lock.rs +60 -0
- salamander_db-0.1.0/salamander/src/log/manifest.rs +98 -0
- salamander_db-0.1.0/salamander/src/log/mod.rs +1150 -0
- salamander_db-0.1.0/salamander/src/log/reader.rs +923 -0
- salamander_db-0.1.0/salamander/src/log/record.rs +169 -0
- salamander_db-0.1.0/salamander/src/log/segment.rs +799 -0
- salamander_db-0.1.0/salamander/src/migration.rs +765 -0
- salamander_db-0.1.0/salamander/src/projection/mod.rs +293 -0
- salamander_db-0.1.0/salamander/src/snapshot.rs +556 -0
- salamander_db-0.1.0/salamander/src/stream.rs +449 -0
- salamander_db-0.1.0/salamander/src/view/indexed.rs +423 -0
- salamander_db-0.1.0/salamander/src/view/mod.rs +75 -0
- salamander_db-0.1.0/salamander-py/Cargo.lock +307 -0
- salamander_db-0.1.0/salamander-py/Cargo.toml +24 -0
- salamander_db-0.1.0/salamander-py/README.md +80 -0
- salamander_db-0.1.0/salamander-py/src/lib.rs +560 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
[workspace]
|
|
2
|
+
resolver = "2"
|
|
3
|
+
members = ["salamander", "salamander-demo"]
|
|
4
|
+
# The Python extension is built by maturin, not the main gate — it links
|
|
5
|
+
# libpython and only makes sense inside a Python build. Excluding it keeps
|
|
6
|
+
# `cargo build/test/clippy` at the root pure Rust.
|
|
7
|
+
exclude = ["salamander-py"]
|
|
8
|
+
|
|
9
|
+
[workspace.package]
|
|
10
|
+
edition = "2021"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
repository = "https://github.com/rdelprete/salamander-db"
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: salamander-db
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Classifier: Programming Language :: Rust
|
|
5
|
+
Classifier: Programming Language :: Python :: 3
|
|
6
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
7
|
+
Summary: SQLite for event-sourced state — Python bindings for salamander-db.
|
|
8
|
+
Keywords: event-sourcing,embedded,database,agent
|
|
9
|
+
License: MIT
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
12
|
+
Project-URL: Repository, https://github.com/rdelprete/salamander-db
|
|
13
|
+
|
|
14
|
+
# salamander-db (Python)
|
|
15
|
+
|
|
16
|
+
Native Python bindings for [salamander-db](../) — **SQLite for event-sourced
|
|
17
|
+
state**. This is a compiled extension (via [PyO3](https://pyo3.rs) +
|
|
18
|
+
[maturin](https://www.maturin.rs)), so it embeds the engine *in your process*
|
|
19
|
+
exactly like Python's built-in `sqlite3` module wraps `libsqlite3`: you open
|
|
20
|
+
one handle and reuse it. No server, no subprocess.
|
|
21
|
+
|
|
22
|
+
The binding is a translation layer over the safe, non-generic Rust `Engine`.
|
|
23
|
+
Database ownership and cross-thread sequencing live in the core; Python does
|
|
24
|
+
not contain storage, replay, branching, or projection algorithms.
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
import salamander
|
|
28
|
+
|
|
29
|
+
db = salamander.open("./mem", commit_every_count=8) # one in-process handle
|
|
30
|
+
db.append("session-1", {"kind": "user_msg", "text": "hi"})
|
|
31
|
+
db.append("session-1", {"kind": "tool_call", "tool": "grep", "args": {"q": "500"}})
|
|
32
|
+
db.commit()
|
|
33
|
+
|
|
34
|
+
for ev in db.replay("session-1"):
|
|
35
|
+
print(ev["offset"], ev["body"]) # -> plain dicts, nested fields intact
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The open handle owns the single-writer lock, any registered views, and the
|
|
39
|
+
group-commit state — which is why it must be long-lived and in-process, not
|
|
40
|
+
re-created per call.
|
|
41
|
+
|
|
42
|
+
## Build it
|
|
43
|
+
|
|
44
|
+
Requires a Rust toolchain and Python ≥ 3.9.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
48
|
+
pip install maturin pytest
|
|
49
|
+
maturin develop -m salamander-py/Cargo.toml # compiles + installs into the venv
|
|
50
|
+
python examples/py/quickstart.py
|
|
51
|
+
python -m pytest examples/py/test_roundtrip.py -v
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
On a very new Python that the pinned PyO3 doesn't recognize yet, prefix the
|
|
55
|
+
build with `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` (the abi3 stable ABI is
|
|
56
|
+
forward-compatible).
|
|
57
|
+
|
|
58
|
+
## API
|
|
59
|
+
|
|
60
|
+
| Python | Engine |
|
|
61
|
+
|---|---|
|
|
62
|
+
| `salamander.open(path, commit_every_bytes=, commit_every_count=, commit_every_millis=)` | `JsonDb::open_with_policy` |
|
|
63
|
+
| `db.append(namespace, event: dict) -> int` | `append` (payload = JSON) |
|
|
64
|
+
| `db.append_branch(branch, namespace, event) -> int` | append to an engine branch |
|
|
65
|
+
| `db.commit() -> int` | `commit` (fsync) |
|
|
66
|
+
| `db.head() -> int` | `head` |
|
|
67
|
+
| `db.uncommitted_count() -> int` | group-commit tally |
|
|
68
|
+
| `db.replay(namespace, start=0, end=None) -> list[dict]` | `replay` (literal) |
|
|
69
|
+
| `db.register_view(name, key, indexes={}, where_field=, where_value=)` | register an `IndexedView` |
|
|
70
|
+
| `db.view(name) -> View` | typed query handle |
|
|
71
|
+
| `db.deregister_view(name) -> bool` | `deregister` |
|
|
72
|
+
| `db.fork(namespace, at) -> str` | create branch (returns branch name) |
|
|
73
|
+
| `db.history(namespace) -> list[dict]` | default-branch replay |
|
|
74
|
+
| `db.branch_history(branch, namespace) -> list[dict]` | inherited branch replay |
|
|
75
|
+
|
|
76
|
+
`View` handles support `.get(key)`, `.by(index, key)`, `.range(lo, hi)`,
|
|
77
|
+
`.prefix(p)`, `.len()`.
|
|
78
|
+
|
|
79
|
+
Payloads are any JSON-able Python value (`dict`/`list`/`str`/`int`/`float`/
|
|
80
|
+
`bool`/`None`), converted to/from `serde_json::Value` at the boundary. Views
|
|
81
|
+
are declared by field name (primary `key`, `indexes` mapping name→field, an
|
|
82
|
+
optional `where_field`/`where_value` filter) — no per-event Python callback
|
|
83
|
+
crosses the FFI boundary.
|
|
84
|
+
|
|
85
|
+
A LangGraph checkpointer that survives process restarts ships in
|
|
86
|
+
[`examples/py/`](../examples/py); an MCP server is on the
|
|
87
|
+
[roadmap](../ROADMAP.md). Branch lifecycle and ancestry use the same engine
|
|
88
|
+
catalog as Rust:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
ancestry = db.branch_ancestry(branch_name) # root-first metadata dictionaries
|
|
92
|
+
archived = db.archive_branch(branch_name) # history remains readable
|
|
93
|
+
```
|
|
94
|
+
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# salamander-db (Python)
|
|
2
|
+
|
|
3
|
+
Native Python bindings for [salamander-db](../) — **SQLite for event-sourced
|
|
4
|
+
state**. This is a compiled extension (via [PyO3](https://pyo3.rs) +
|
|
5
|
+
[maturin](https://www.maturin.rs)), so it embeds the engine *in your process*
|
|
6
|
+
exactly like Python's built-in `sqlite3` module wraps `libsqlite3`: you open
|
|
7
|
+
one handle and reuse it. No server, no subprocess.
|
|
8
|
+
|
|
9
|
+
The binding is a translation layer over the safe, non-generic Rust `Engine`.
|
|
10
|
+
Database ownership and cross-thread sequencing live in the core; Python does
|
|
11
|
+
not contain storage, replay, branching, or projection algorithms.
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
import salamander
|
|
15
|
+
|
|
16
|
+
db = salamander.open("./mem", commit_every_count=8) # one in-process handle
|
|
17
|
+
db.append("session-1", {"kind": "user_msg", "text": "hi"})
|
|
18
|
+
db.append("session-1", {"kind": "tool_call", "tool": "grep", "args": {"q": "500"}})
|
|
19
|
+
db.commit()
|
|
20
|
+
|
|
21
|
+
for ev in db.replay("session-1"):
|
|
22
|
+
print(ev["offset"], ev["body"]) # -> plain dicts, nested fields intact
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The open handle owns the single-writer lock, any registered views, and the
|
|
26
|
+
group-commit state — which is why it must be long-lived and in-process, not
|
|
27
|
+
re-created per call.
|
|
28
|
+
|
|
29
|
+
## Build it
|
|
30
|
+
|
|
31
|
+
Requires a Rust toolchain and Python ≥ 3.9.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
35
|
+
pip install maturin pytest
|
|
36
|
+
maturin develop -m salamander-py/Cargo.toml # compiles + installs into the venv
|
|
37
|
+
python examples/py/quickstart.py
|
|
38
|
+
python -m pytest examples/py/test_roundtrip.py -v
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
On a very new Python that the pinned PyO3 doesn't recognize yet, prefix the
|
|
42
|
+
build with `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` (the abi3 stable ABI is
|
|
43
|
+
forward-compatible).
|
|
44
|
+
|
|
45
|
+
## API
|
|
46
|
+
|
|
47
|
+
| Python | Engine |
|
|
48
|
+
|---|---|
|
|
49
|
+
| `salamander.open(path, commit_every_bytes=, commit_every_count=, commit_every_millis=)` | `JsonDb::open_with_policy` |
|
|
50
|
+
| `db.append(namespace, event: dict) -> int` | `append` (payload = JSON) |
|
|
51
|
+
| `db.append_branch(branch, namespace, event) -> int` | append to an engine branch |
|
|
52
|
+
| `db.commit() -> int` | `commit` (fsync) |
|
|
53
|
+
| `db.head() -> int` | `head` |
|
|
54
|
+
| `db.uncommitted_count() -> int` | group-commit tally |
|
|
55
|
+
| `db.replay(namespace, start=0, end=None) -> list[dict]` | `replay` (literal) |
|
|
56
|
+
| `db.register_view(name, key, indexes={}, where_field=, where_value=)` | register an `IndexedView` |
|
|
57
|
+
| `db.view(name) -> View` | typed query handle |
|
|
58
|
+
| `db.deregister_view(name) -> bool` | `deregister` |
|
|
59
|
+
| `db.fork(namespace, at) -> str` | create branch (returns branch name) |
|
|
60
|
+
| `db.history(namespace) -> list[dict]` | default-branch replay |
|
|
61
|
+
| `db.branch_history(branch, namespace) -> list[dict]` | inherited branch replay |
|
|
62
|
+
|
|
63
|
+
`View` handles support `.get(key)`, `.by(index, key)`, `.range(lo, hi)`,
|
|
64
|
+
`.prefix(p)`, `.len()`.
|
|
65
|
+
|
|
66
|
+
Payloads are any JSON-able Python value (`dict`/`list`/`str`/`int`/`float`/
|
|
67
|
+
`bool`/`None`), converted to/from `serde_json::Value` at the boundary. Views
|
|
68
|
+
are declared by field name (primary `key`, `indexes` mapping name→field, an
|
|
69
|
+
optional `where_field`/`where_value` filter) — no per-event Python callback
|
|
70
|
+
crosses the FFI boundary.
|
|
71
|
+
|
|
72
|
+
A LangGraph checkpointer that survives process restarts ships in
|
|
73
|
+
[`examples/py/`](../examples/py); an MCP server is on the
|
|
74
|
+
[roadmap](../ROADMAP.md). Branch lifecycle and ancestry use the same engine
|
|
75
|
+
catalog as Rust:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
ancestry = db.branch_ancestry(branch_name) # root-first metadata dictionaries
|
|
79
|
+
archived = db.archive_branch(branch_name) # history remains readable
|
|
80
|
+
```
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.5,<2.0"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "salamander-db"
|
|
7
|
+
# Sourced from salamander-py/Cargo.toml by maturin — one place to bump.
|
|
8
|
+
dynamic = ["version"]
|
|
9
|
+
description = "SQLite for event-sourced state — Python bindings for salamander-db."
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
license = { text = "MIT" }
|
|
13
|
+
keywords = ["event-sourcing", "embedded", "database", "agent"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Rust",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Repository = "https://github.com/rdelprete/salamander-db"
|
|
22
|
+
|
|
23
|
+
[tool.maturin]
|
|
24
|
+
# The Rust cdylib lib is named `salamander`, which is the Python import name;
|
|
25
|
+
# the distribution ("salamander-db") differs, mirroring the Rust package/lib
|
|
26
|
+
# split. abi3 gives one wheel across CPython 3.9+.
|
|
27
|
+
module-name = "salamander"
|
|
28
|
+
features = ["pyo3/extension-module"]
|
|
29
|
+
manifest-path = "salamander-py/Cargo.toml"
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "salamander-db"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition.workspace = true
|
|
5
|
+
license.workspace = true
|
|
6
|
+
repository.workspace = true
|
|
7
|
+
description = "Embedded event-sourcing engine with instant recovery — the append-only log is the only durable structure."
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
documentation = "https://docs.rs/salamander-db"
|
|
10
|
+
homepage = "https://github.com/rdelprete/salamander-db"
|
|
11
|
+
keywords = ["event-sourcing", "embedded", "database", "wal", "agent"]
|
|
12
|
+
categories = ["database-implementations", "data-structures"]
|
|
13
|
+
rust-version = "1.90"
|
|
14
|
+
# Integration tests and their fixtures are development-only; keep them out
|
|
15
|
+
# of the published crate (the lib, examples, and benches ship).
|
|
16
|
+
exclude = ["tests/"]
|
|
17
|
+
|
|
18
|
+
[lib]
|
|
19
|
+
name = "salamander"
|
|
20
|
+
path = "src/lib.rs"
|
|
21
|
+
|
|
22
|
+
[dependencies]
|
|
23
|
+
crc32c = "0.6"
|
|
24
|
+
serde = { version = "1", features = ["derive"] }
|
|
25
|
+
serde_json = "1"
|
|
26
|
+
bincode = "1.3"
|
|
27
|
+
thiserror = "1"
|
|
28
|
+
|
|
29
|
+
[dev-dependencies]
|
|
30
|
+
proptest = "1"
|
|
31
|
+
criterion = "0.5"
|
|
32
|
+
tempfile = "3"
|
|
33
|
+
|
|
34
|
+
[[bench]]
|
|
35
|
+
name = "open_time"
|
|
36
|
+
harness = false
|
|
37
|
+
|
|
38
|
+
[[bench]]
|
|
39
|
+
name = "view_apply"
|
|
40
|
+
harness = false
|
|
41
|
+
|
|
42
|
+
[[bench]]
|
|
43
|
+
name = "replay"
|
|
44
|
+
harness = false
|
|
45
|
+
|
|
46
|
+
[[bench]]
|
|
47
|
+
name = "snapshots"
|
|
48
|
+
harness = false
|
|
49
|
+
|
|
50
|
+
[[bench]]
|
|
51
|
+
name = "instant_recovery"
|
|
52
|
+
harness = false
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Raffaele Del Prete
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# salamander-db
|
|
2
|
+
|
|
3
|
+
**SQLite for event-sourced state — built first for agent memory.**
|
|
4
|
+
|
|
5
|
+
An embedded event-sourcing engine. The append-only log is the only durable
|
|
6
|
+
structure; everything else is a rebuildable projection. Replay, time-travel,
|
|
7
|
+
and fork are first-class: rewind any session to step N, branch it, and run
|
|
8
|
+
two futures against the same history.
|
|
9
|
+
|
|
10
|
+
Second story: **crash-proof application state.** The torn-tail rule makes
|
|
11
|
+
"state that never logically existed" impossible after a kill — and the full
|
|
12
|
+
history comes free.
|
|
13
|
+
|
|
14
|
+
```rust
|
|
15
|
+
use salamander::agent::{EventBody, KvProjection};
|
|
16
|
+
use salamander::{AgentDb, Projection};
|
|
17
|
+
|
|
18
|
+
let mut db = AgentDb::open("./mydata")?;
|
|
19
|
+
db.append("session-1", EventBody::Put { key: "k".into(), value: b"v".to_vec() })?;
|
|
20
|
+
db.commit()?; // fsync; durable
|
|
21
|
+
|
|
22
|
+
// Everything else rebuilds from the log:
|
|
23
|
+
let kv: KvProjection = db.projection()?; // full replay
|
|
24
|
+
let past: KvProjection = db.view_at(1)?; // state as of offset 1 (time-travel)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Features
|
|
28
|
+
|
|
29
|
+
- **Segmented append-only log** — CRC32C framing, torn-tail truncation on
|
|
30
|
+
open, atomic manifest, single-writer lock.
|
|
31
|
+
- **Projections** — deterministic folds of the log; `KvProjection`,
|
|
32
|
+
`SessionProjection`, or your own via the `Projection` trait.
|
|
33
|
+
- **Time-travel & fork** — `view_at(n)`; `fork(ns, n)` branches a session
|
|
34
|
+
while the log stays linear.
|
|
35
|
+
- **Payload-generic** — `Salamander<B>` over any serde payload; a provided
|
|
36
|
+
`agent` vocabulary, `JsonDb` for dynamic JSON, or bring your own enum.
|
|
37
|
+
- **Query layer** — live registered views with secondary indexes:
|
|
38
|
+
`get` / `range` / `prefix` / `by`, maintained incrementally (never stale).
|
|
39
|
+
- **Group commit** — combinable byte/count/time commit policies.
|
|
40
|
+
- **Crash-tested** — a `kill -9` harness asserts the core invariant across
|
|
41
|
+
thousands of crashes.
|
|
42
|
+
|
|
43
|
+
## Status
|
|
44
|
+
|
|
45
|
+
Phase 1 core + Phase 1.5 (payload-generic engine, query layer, group commit,
|
|
46
|
+
dynamic-JSON payloads) complete. Single-writer, embedded, in-memory
|
|
47
|
+
projections (persistence is Phase 2). Not multi-writer, not a SQL/query
|
|
48
|
+
language, not a vector store — by design.
|
|
49
|
+
|
|
50
|
+
Roadmap, changelog, and runnable examples:
|
|
51
|
+
<https://github.com/rdelprete/salamander-db>
|
|
52
|
+
|
|
53
|
+
## License
|
|
54
|
+
|
|
55
|
+
MIT
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
use std::collections::BTreeMap;
|
|
2
|
+
|
|
3
|
+
use criterion::{black_box, criterion_group, criterion_main, Criterion};
|
|
4
|
+
use salamander::{
|
|
5
|
+
DurabilityDto, Engine, EngineAppendBatch, EngineOptions, EventData, ExpectedRevisionDto,
|
|
6
|
+
PayloadCodec, QueryConsistency, QueryDefinition, QueryOperation,
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
fn fixture(events: usize) -> tempfile::TempDir {
|
|
10
|
+
let dir = tempfile::tempdir().unwrap();
|
|
11
|
+
let engine = Engine::open(EngineOptions::new(dir.path())).unwrap();
|
|
12
|
+
for index in 0..events {
|
|
13
|
+
engine
|
|
14
|
+
.append(EngineAppendBatch {
|
|
15
|
+
branch_id: [0; 16],
|
|
16
|
+
stream: format!("s{}", index % 128),
|
|
17
|
+
expected: ExpectedRevisionDto::Any,
|
|
18
|
+
idempotency_key: None,
|
|
19
|
+
durability: DurabilityDto::Buffered,
|
|
20
|
+
events: vec![EventData {
|
|
21
|
+
event_id: None,
|
|
22
|
+
event_type: "row".into(),
|
|
23
|
+
schema_version: 1,
|
|
24
|
+
metadata: BTreeMap::new(),
|
|
25
|
+
codec: PayloadCodec::Json,
|
|
26
|
+
payload: serde_json::to_vec(&serde_json::json!({"id": format!("k{index}")}))
|
|
27
|
+
.unwrap(),
|
|
28
|
+
}],
|
|
29
|
+
})
|
|
30
|
+
.unwrap();
|
|
31
|
+
}
|
|
32
|
+
engine.commit().unwrap();
|
|
33
|
+
engine
|
|
34
|
+
.register_partitioned_query(
|
|
35
|
+
"rows".into(),
|
|
36
|
+
QueryDefinition {
|
|
37
|
+
key_field: "id".into(),
|
|
38
|
+
indexes: BTreeMap::new(),
|
|
39
|
+
filter: None,
|
|
40
|
+
},
|
|
41
|
+
64,
|
|
42
|
+
)
|
|
43
|
+
.unwrap();
|
|
44
|
+
engine.close().unwrap();
|
|
45
|
+
dir
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
fn bench(c: &mut Criterion) {
|
|
49
|
+
let fixture = fixture(100_000);
|
|
50
|
+
c.bench_function("instant_recovery/open_catalog_only", |b| {
|
|
51
|
+
b.iter(|| {
|
|
52
|
+
let engine = Engine::open(EngineOptions::new(fixture.path())).unwrap();
|
|
53
|
+
black_box(engine.head().unwrap());
|
|
54
|
+
engine.close().unwrap();
|
|
55
|
+
})
|
|
56
|
+
});
|
|
57
|
+
c.bench_function("instant_recovery/heal_one_partition", |b| {
|
|
58
|
+
b.iter(|| {
|
|
59
|
+
let engine = Engine::open(EngineOptions::new(fixture.path())).unwrap();
|
|
60
|
+
let handle = engine.query_named("rows".into()).unwrap();
|
|
61
|
+
black_box(
|
|
62
|
+
engine
|
|
63
|
+
.query_partitions(
|
|
64
|
+
handle,
|
|
65
|
+
vec![0],
|
|
66
|
+
QueryOperation::Len,
|
|
67
|
+
QueryConsistency::RequireHead,
|
|
68
|
+
)
|
|
69
|
+
.unwrap(),
|
|
70
|
+
);
|
|
71
|
+
engine.close().unwrap();
|
|
72
|
+
})
|
|
73
|
+
});
|
|
74
|
+
c.bench_function("instant_recovery/heal_all_partitions", |b| {
|
|
75
|
+
b.iter(|| {
|
|
76
|
+
let engine = Engine::open(EngineOptions::new(fixture.path())).unwrap();
|
|
77
|
+
let handle = engine.query_named("rows".into()).unwrap();
|
|
78
|
+
black_box(engine.query(handle, QueryOperation::Len).unwrap());
|
|
79
|
+
engine.close().unwrap();
|
|
80
|
+
})
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
criterion_group!(benches, bench);
|
|
85
|
+
criterion_main!(benches);
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
//! M5 — IMPLEMENTATION.md Step 7 / DESIGN.md §9.
|
|
2
|
+
//!
|
|
3
|
+
//! Criterion bench: cold-start cost vs. log size — the "baseline villain"
|
|
4
|
+
//! that motivates Phase 2 (DESIGN.md §9, §8). Two measurements per size:
|
|
5
|
+
//!
|
|
6
|
+
//! - `open_only` — `AgentDb::open`, which recovers the *log* (scans the
|
|
7
|
+
//! active segment for a torn tail) but replays no projections. Roughly
|
|
8
|
+
//! flat in log size: only the last segment is touched.
|
|
9
|
+
//! - `open_and_replay` — `open` plus a full-log `KvProjection` rebuild.
|
|
10
|
+
//! Linear in log size — this is the cost Phase 1 pays to reach usable
|
|
11
|
+
//! derived state, and what instant recovery (Phase 2) exists to erase.
|
|
12
|
+
//!
|
|
13
|
+
//! Sizes come from the `SALAMANDER_BENCH_SIZES` env var (comma-separated
|
|
14
|
+
//! event counts); the default is a quick `100000,1000000`. The canonical
|
|
15
|
+
//! M5 run is:
|
|
16
|
+
//!
|
|
17
|
+
//! ```text
|
|
18
|
+
//! SALAMANDER_BENCH_SIZES=1000000,10000000,50000000 cargo bench --bench open_time
|
|
19
|
+
//! ```
|
|
20
|
+
//!
|
|
21
|
+
//! Fixtures are generated once per size, outside the measured loop.
|
|
22
|
+
|
|
23
|
+
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
|
|
24
|
+
use salamander::agent::{EventBody, KvProjection};
|
|
25
|
+
use salamander::{AgentDb, Projection};
|
|
26
|
+
use tempfile::TempDir;
|
|
27
|
+
|
|
28
|
+
const NAMESPACE: &str = "bench";
|
|
29
|
+
|
|
30
|
+
fn sizes() -> Vec<u64> {
|
|
31
|
+
match std::env::var("SALAMANDER_BENCH_SIZES") {
|
|
32
|
+
Ok(s) => s
|
|
33
|
+
.split(',')
|
|
34
|
+
.filter_map(|p| p.trim().parse::<u64>().ok())
|
|
35
|
+
.filter(|&n| n > 0)
|
|
36
|
+
.collect(),
|
|
37
|
+
Err(_) => vec![100_000, 1_000_000],
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/// Build a fixture directory holding `n` committed events. One constant key
|
|
42
|
+
/// keeps the projection's map size flat, so `open_and_replay` measures the
|
|
43
|
+
/// pure per-event decode+apply cost rather than BTree growth.
|
|
44
|
+
fn populate(n: u64) -> TempDir {
|
|
45
|
+
let dir = tempfile::tempdir().expect("tempdir");
|
|
46
|
+
{
|
|
47
|
+
let mut db = AgentDb::open(dir.path()).expect("open");
|
|
48
|
+
for i in 0..n {
|
|
49
|
+
db.append(
|
|
50
|
+
NAMESPACE,
|
|
51
|
+
EventBody::Put {
|
|
52
|
+
key: "bench".to_string(),
|
|
53
|
+
value: i.to_le_bytes().to_vec(),
|
|
54
|
+
},
|
|
55
|
+
)
|
|
56
|
+
.expect("append");
|
|
57
|
+
}
|
|
58
|
+
db.commit().expect("commit");
|
|
59
|
+
} // drop releases the single-writer lock so the bench can reopen freely
|
|
60
|
+
dir
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
fn bench_open_time(c: &mut Criterion) {
|
|
64
|
+
// Generate every fixture up front; each stays alive (as a TempDir) for
|
|
65
|
+
// the whole run and is cleaned up when this Vec drops.
|
|
66
|
+
let fixtures: Vec<(u64, TempDir)> = sizes().into_iter().map(|n| (n, populate(n))).collect();
|
|
67
|
+
|
|
68
|
+
let mut group = c.benchmark_group("open_time");
|
|
69
|
+
// Opening a large log is expensive; 10 is criterion's minimum sample
|
|
70
|
+
// count and keeps a 50M-event run from taking hours.
|
|
71
|
+
group.sample_size(10);
|
|
72
|
+
|
|
73
|
+
for (n, dir) in &fixtures {
|
|
74
|
+
group.throughput(Throughput::Elements(*n));
|
|
75
|
+
|
|
76
|
+
group.bench_with_input(BenchmarkId::new("open_only", n), dir, |b, dir| {
|
|
77
|
+
b.iter(|| {
|
|
78
|
+
let db = AgentDb::open(dir.path()).expect("open");
|
|
79
|
+
black_box(db.head())
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
group.bench_with_input(BenchmarkId::new("open_and_replay", n), dir, |b, dir| {
|
|
84
|
+
b.iter(|| {
|
|
85
|
+
let db = AgentDb::open(dir.path()).expect("open");
|
|
86
|
+
let kv = db.projection::<KvProjection>().expect("projection");
|
|
87
|
+
black_box(kv.state().len())
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
group.finish();
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
criterion_group!(benches, bench_open_time);
|
|
96
|
+
criterion_main!(benches);
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
//! WP-04 — streaming-reader throughput and selectivity.
|
|
2
|
+
//!
|
|
3
|
+
//! Three measurements per size:
|
|
4
|
+
//!
|
|
5
|
+
//! - `full_scan` — stream every record through the bounded reader
|
|
6
|
+
//! (replaces the old materializing scan; linear, but O(1) memory).
|
|
7
|
+
//! - `single_stream` — a `StreamSelector::Streams` plan for one of
|
|
8
|
+
//! `STREAM_FAN` streams. With sidecar postings this touches only the
|
|
9
|
+
//! selected records' segments; the gap to `full_scan / STREAM_FAN`
|
|
10
|
+
//! measures the selective-read win WP-09 healing relies on.
|
|
11
|
+
//! - `tail_seek` — read the final 100 records. Sub-linear thanks to
|
|
12
|
+
//! segment binary search plus in-segment seek points.
|
|
13
|
+
//!
|
|
14
|
+
//! Sizes come from `SALAMANDER_BENCH_SIZES` (comma-separated counts);
|
|
15
|
+
//! default `100000,1000000`. The spec's canonical runs:
|
|
16
|
+
//!
|
|
17
|
+
//! ```text
|
|
18
|
+
//! SALAMANDER_BENCH_SIZES=1000000 cargo bench --bench replay
|
|
19
|
+
//! SALAMANDER_BENCH_SIZES=10000000 cargo bench --bench replay # 10M physical
|
|
20
|
+
//! ```
|
|
21
|
+
//!
|
|
22
|
+
//! (The 100M *planning* case is a unit test — `intersecting_range` over
|
|
23
|
+
//! synthetic segment metadata — since it measures index arithmetic, not
|
|
24
|
+
//! I/O.)
|
|
25
|
+
|
|
26
|
+
use std::ops::Bound;
|
|
27
|
+
|
|
28
|
+
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
|
|
29
|
+
use salamander::agent::EventBody;
|
|
30
|
+
use salamander::{
|
|
31
|
+
AgentDb, AppendRequest, BranchId, Durability, EventType, ExpectedRevision, NewEvent,
|
|
32
|
+
RecordReader, ReplayPlan, StreamId, StreamName, StreamSelector,
|
|
33
|
+
};
|
|
34
|
+
use tempfile::TempDir;
|
|
35
|
+
|
|
36
|
+
const STREAM_FAN: u64 = 16;
|
|
37
|
+
|
|
38
|
+
fn sizes() -> Vec<u64> {
|
|
39
|
+
match std::env::var("SALAMANDER_BENCH_SIZES") {
|
|
40
|
+
Ok(s) => s
|
|
41
|
+
.split(',')
|
|
42
|
+
.filter_map(|p| p.trim().parse::<u64>().ok())
|
|
43
|
+
.filter(|&n| n > 0)
|
|
44
|
+
.collect(),
|
|
45
|
+
Err(_) => vec![100_000, 1_000_000],
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/// `n` events spread round-robin over `STREAM_FAN` streams, committed in
|
|
50
|
+
/// batches of 64 to keep fixture generation fast. Returns the fixture dir
|
|
51
|
+
/// and one stream's id for selective plans.
|
|
52
|
+
fn populate(n: u64) -> (TempDir, StreamId) {
|
|
53
|
+
let dir = tempfile::tempdir().expect("tempdir");
|
|
54
|
+
let mut chosen = None;
|
|
55
|
+
{
|
|
56
|
+
let mut db = AgentDb::open(dir.path()).expect("open");
|
|
57
|
+
let mut appended = 0u64;
|
|
58
|
+
while appended < n {
|
|
59
|
+
let stream = appended % STREAM_FAN;
|
|
60
|
+
let batch = (n - appended).min(64);
|
|
61
|
+
let receipt = db
|
|
62
|
+
.append_batch(AppendRequest {
|
|
63
|
+
branch: BranchId::ZERO,
|
|
64
|
+
stream: StreamName::new(format!("stream-{stream}")).expect("name"),
|
|
65
|
+
expected: ExpectedRevision::Any,
|
|
66
|
+
idempotency_key: None,
|
|
67
|
+
events: (0..batch)
|
|
68
|
+
.map(|i| {
|
|
69
|
+
NewEvent::new(
|
|
70
|
+
EventType::new("bench.put").expect("type"),
|
|
71
|
+
EventBody::Put {
|
|
72
|
+
key: "k".into(),
|
|
73
|
+
value: (appended + i).to_le_bytes().to_vec(),
|
|
74
|
+
},
|
|
75
|
+
)
|
|
76
|
+
})
|
|
77
|
+
.collect(),
|
|
78
|
+
durability: Durability::Buffered,
|
|
79
|
+
})
|
|
80
|
+
.expect("append");
|
|
81
|
+
if stream == 0 {
|
|
82
|
+
chosen = Some(receipt.stream_id);
|
|
83
|
+
}
|
|
84
|
+
appended += batch;
|
|
85
|
+
}
|
|
86
|
+
db.commit().expect("commit");
|
|
87
|
+
}
|
|
88
|
+
(dir, chosen.expect("at least one batch"))
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
fn drain(db: &AgentDb, plan: ReplayPlan) -> u64 {
|
|
92
|
+
let mut reader = db.read(plan).expect("plan");
|
|
93
|
+
let mut count = 0u64;
|
|
94
|
+
while reader.next().expect("read").is_some() {
|
|
95
|
+
count += 1;
|
|
96
|
+
}
|
|
97
|
+
count
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
fn bench_replay(c: &mut Criterion) {
|
|
101
|
+
let mut group = c.benchmark_group("replay");
|
|
102
|
+
group.sample_size(10);
|
|
103
|
+
|
|
104
|
+
for n in sizes() {
|
|
105
|
+
let (dir, stream) = populate(n);
|
|
106
|
+
let db = AgentDb::open(dir.path()).expect("reopen");
|
|
107
|
+
group.throughput(Throughput::Elements(n));
|
|
108
|
+
|
|
109
|
+
group.bench_with_input(BenchmarkId::new("full_scan", n), &n, |b, _| {
|
|
110
|
+
b.iter(|| black_box(drain(&db, ReplayPlan::default())))
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
group.bench_with_input(BenchmarkId::new("single_stream", n), &n, |b, _| {
|
|
114
|
+
b.iter(|| {
|
|
115
|
+
black_box(drain(
|
|
116
|
+
&db,
|
|
117
|
+
ReplayPlan {
|
|
118
|
+
streams: StreamSelector::Streams(vec![stream]),
|
|
119
|
+
..ReplayPlan::default()
|
|
120
|
+
},
|
|
121
|
+
))
|
|
122
|
+
})
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
group.bench_with_input(BenchmarkId::new("tail_seek", n), &n, |b, _| {
|
|
126
|
+
b.iter(|| {
|
|
127
|
+
black_box(drain(
|
|
128
|
+
&db,
|
|
129
|
+
ReplayPlan {
|
|
130
|
+
from: Bound::Included(n.saturating_sub(100)),
|
|
131
|
+
..ReplayPlan::default()
|
|
132
|
+
},
|
|
133
|
+
))
|
|
134
|
+
})
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
group.finish();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
criterion_group!(benches, bench_replay);
|
|
141
|
+
criterion_main!(benches);
|