dnhydb 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dnhydb-0.2.0 → dnhydb-0.3.0}/Cargo.lock +2 -2
- {dnhydb-0.2.0 → dnhydb-0.3.0}/Cargo.toml +1 -1
- dnhydb-0.3.0/PKG-INFO +226 -0
- dnhydb-0.3.0/README.md +190 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/Cargo.toml +4 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/address.rs +3 -92
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/codec.rs +35 -70
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/buffer.rs +59 -24
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/catalog.rs +7 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/columns.rs +116 -19
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/dense.rs +44 -12
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/index.rs +63 -8
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/interval.rs +3 -59
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/iter.rs +3 -1
- dnhydb-0.3.0/crates/dnhydb-core/src/db/maint.rs +1186 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/meta.rs +16 -3
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/mod.rs +468 -255
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/read.rs +7 -1
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/sparse.rs +73 -15
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/db/vindex.rs +2 -1
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/error.rs +19 -2
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/format.rs +3 -25
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/lib.rs +1 -1
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/platform.rs +218 -21
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/schema.rs +6 -34
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/storage.rs +59 -3
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/u256.rs +3 -57
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/val.rs +27 -51
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/src/varint.rs +3 -65
- dnhydb-0.3.0/crates/dnhydb-core/src/zlib.rs +37 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-py/src/conv.rs +91 -10
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-py/src/lib.rs +115 -26
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-py/src/numpy_io.rs +48 -32
- {dnhydb-0.2.0 → dnhydb-0.3.0}/pyproject.toml +12 -3
- {dnhydb-0.2.0 → dnhydb-0.3.0}/python/dnhydb/__init__.py +3 -7
- {dnhydb-0.2.0 → dnhydb-0.3.0}/python/dnhydb/cli.py +34 -9
- dnhydb-0.2.0/PKG-INFO +0 -197
- dnhydb-0.2.0/README.md +0 -168
- dnhydb-0.2.0/crates/dnhydb-core/src/db/maint.rs +0 -672
- dnhydb-0.2.0/crates/dnhydb-core/src/zlib.rs +0 -25
- dnhydb-0.2.0/crates/dnhydb-core/tests/common/mod.rs +0 -35
- dnhydb-0.2.0/crates/dnhydb-core/tests/core.rs +0 -579
- dnhydb-0.2.0/crates/dnhydb-core/tests/model.rs +0 -285
- dnhydb-0.2.0/crates/dnhydb-core/tests/multiprocess.rs +0 -202
- dnhydb-0.2.0/crates/dnhydb-core/tests/robustness.rs +0 -180
- {dnhydb-0.2.0 → dnhydb-0.3.0}/LICENSE.md +0 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/THIRD-PARTY-LICENSES.txt +0 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/LICENSE.md +0 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-core/examples/bench.rs +0 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-py/Cargo.toml +0 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-py/LICENSE.md +0 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/crates/dnhydb-py/src/exceptions.rs +0 -0
- {dnhydb-0.2.0 → dnhydb-0.3.0}/python/dnhydb/__main__.py +0 -0
|
@@ -25,7 +25,7 @@ dependencies = [
|
|
|
25
25
|
|
|
26
26
|
[[package]]
|
|
27
27
|
name = "dnhydb-core"
|
|
28
|
-
version = "0.
|
|
28
|
+
version = "0.3.0"
|
|
29
29
|
dependencies = [
|
|
30
30
|
"crc32fast",
|
|
31
31
|
"libc",
|
|
@@ -35,7 +35,7 @@ dependencies = [
|
|
|
35
35
|
|
|
36
36
|
[[package]]
|
|
37
37
|
name = "dnhydb-py"
|
|
38
|
-
version = "0.
|
|
38
|
+
version = "0.3.0"
|
|
39
39
|
dependencies = [
|
|
40
40
|
"dnhydb-core",
|
|
41
41
|
"pyo3",
|
dnhydb-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dnhydb
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Classifier: Development Status :: 3 - Alpha
|
|
5
|
+
Classifier: License :: Other/Proprietary License
|
|
6
|
+
Classifier: Intended Audience :: Developers
|
|
7
|
+
Classifier: Intended Audience :: Science/Research
|
|
8
|
+
Classifier: Operating System :: MacOS
|
|
9
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
10
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
11
|
+
Classifier: Programming Language :: Rust
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
17
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
18
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
19
|
+
Requires-Dist: numpy>=1.22 ; extra == 'numpy'
|
|
20
|
+
Requires-Dist: pytest>=7 ; extra == 'test'
|
|
21
|
+
Requires-Dist: numpy>=1.22 ; extra == 'test'
|
|
22
|
+
Provides-Extra: numpy
|
|
23
|
+
Provides-Extra: test
|
|
24
|
+
License-File: LICENSE.md
|
|
25
|
+
License-File: THIRD-PARTY-LICENSES.txt
|
|
26
|
+
Summary: dNHyDB (directNHyDB) – general-purpose database with direct addressing (u, v, side); native Rust engine with Python bindings
|
|
27
|
+
Keywords: database,storage-engine,rust,numpy,append-only
|
|
28
|
+
Author-email: Achim Meissner <achim.meissner@nhydb.de>
|
|
29
|
+
Requires-Python: >=3.12
|
|
30
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
31
|
+
Project-URL: Changelog, https://github.com/nhydb-run4/dNHyDB/blob/main/CHANGELOG.md
|
|
32
|
+
Project-URL: Documentation, https://github.com/nhydb-run4/dNHyDB/blob/main/docs/cookbook/PYTHON_COOKBOOK.md
|
|
33
|
+
Project-URL: Issues, https://github.com/nhydb-run4/dNHyDB/issues
|
|
34
|
+
Project-URL: Repository, https://github.com/nhydb-run4/dNHyDB
|
|
35
|
+
|
|
36
|
+
# dNHyDB (directNHyDB)
|
|
37
|
+
|
|
38
|
+
**A compact record store with direct mathematical addressing.**
|
|
39
|
+
|
|
40
|
+
dNHyDB is the streamlined member of the **Dynadron** database family. Every
|
|
41
|
+
value lives in a cell `(u, v, s)`: `u` is the record, `v` the field or
|
|
42
|
+
position, `s` one of two sides. There are no tables and no primary index –
|
|
43
|
+
the address space itself provides order, and each cell has an exact natural
|
|
44
|
+
number `N` that is computed when needed, never stored. The engine is written
|
|
45
|
+
in native Rust and used from Python.
|
|
46
|
+
|
|
47
|
+
[](https://pypi.org/project/dnhydb/)
|
|
48
|
+
· Python 3.12–3.15 · Linux, macOS, Windows ·
|
|
49
|
+
[License: source-available, non-production](#license)
|
|
50
|
+
|
|
51
|
+
**→ [Python Cookbook](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/cookbook/PYTHON_COOKBOOK.md)** –
|
|
52
|
+
task-oriented guide with tested examples for the complete Python API.
|
|
53
|
+
|
|
54
|
+
## The addressing idea
|
|
55
|
+
|
|
56
|
+
`u` and `v` are integers from 1 to 2⁶⁴ − 1, the side `s` is `NL` or `NR`.
|
|
57
|
+
The number of a cell is
|
|
58
|
+
|
|
59
|
+
```text
|
|
60
|
+
G = u + v N = G² − u (side NL) N = G² + v (side NR)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
so `(u, v) = (11, 13)` is `N = 565` on NL and `N = 589` on NR. The mapping is
|
|
64
|
+
exact in both directions (`dnhydb.n_of`, `dnhydb.address_of`); `N` needs up
|
|
65
|
+
to 130 bits and is handled as a Python integer.
|
|
66
|
+
|
|
67
|
+
## Installation
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install dnhydb
|
|
71
|
+
pip install numpy # optional, for the NumPy batch API
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Prebuilt wheels are available for CPython 3.12–3.15 on Linux (x86_64,
|
|
75
|
+
aarch64), macOS (x86_64, arm64) and Windows (x64). No Rust toolchain is
|
|
76
|
+
needed; building from source is described in
|
|
77
|
+
[`BUILDING.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/BUILDING.md).
|
|
78
|
+
|
|
79
|
+
## A first example
|
|
80
|
+
|
|
81
|
+
Create a database, write two records and a free cell, close it, open it
|
|
82
|
+
again as a reader and read:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import os
|
|
86
|
+
import tempfile
|
|
87
|
+
|
|
88
|
+
import dnhydb
|
|
89
|
+
|
|
90
|
+
path = os.path.join(tempfile.mkdtemp(), "cities.dnhydb")
|
|
91
|
+
|
|
92
|
+
# Create: a typed BASE record layout, two records and one free cell.
|
|
93
|
+
with dnhydb.create(path) as db:
|
|
94
|
+
db.define_base([("city", "utf8"), ("population", "u64")])
|
|
95
|
+
db.put_record(1, ["Berlin", 3_685_000])
|
|
96
|
+
db.put_record(2, ["Hamburg", 1_862_000])
|
|
97
|
+
db.put_cell(2, 1_000, "port city")
|
|
98
|
+
# Leaving the block commits and closes the database.
|
|
99
|
+
|
|
100
|
+
db = dnhydb.open_reader(path)
|
|
101
|
+
print(db.get_record(2))
|
|
102
|
+
print(db.get_cell(2, 1_000))
|
|
103
|
+
n = dnhydb.n_of(2, 1)
|
|
104
|
+
print(n, dnhydb.address_of(n))
|
|
105
|
+
db.close()
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
```text
|
|
109
|
+
{'city': 'Hamburg', 'population': 1862000, 'v1000': 'port city'}
|
|
110
|
+
port city
|
|
111
|
+
7 (2, 1, 0)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The database is a single file (`cities.dnhydb`); next to it the engine keeps
|
|
115
|
+
a `.lock` file for the writer and a catalog (`.dnhydb-index`) that speeds up
|
|
116
|
+
opening and is rebuilt automatically. For a backup the `.dnhydb` file is
|
|
117
|
+
enough – see
|
|
118
|
+
[Cookbook, section 17](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/cookbook/PYTHON_COOKBOOK.md#17-files-on-disk-backup-and-restore).
|
|
119
|
+
|
|
120
|
+
## What dNHyDB offers
|
|
121
|
+
|
|
122
|
+
* **Records and cells:** a typed BASE record layout (`define_base`), named
|
|
123
|
+
extensions, free cells anywhere in the address space, deletion with
|
|
124
|
+
tombstones, two sides per cell.
|
|
125
|
+
* **Write path:** write buffer, batches, NumPy column batches, commit policy
|
|
126
|
+
(manual or by thresholds), transactions with rollback, durability `fsync`,
|
|
127
|
+
`full` or `none`.
|
|
128
|
+
* **One writer, many readers:** a reader – also in another process – sees
|
|
129
|
+
the last commit as a consistent snapshot and moves forward with
|
|
130
|
+
`refresh()`.
|
|
131
|
+
* **Crash safety:** append-only file with two CRC-protected superblocks; after
|
|
132
|
+
a crash the last commit is valid, never half a batch.
|
|
133
|
+
* **Reading:** point access, records, iterators that skip gaps, column reads
|
|
134
|
+
into NumPy, a VALUE index for equality and range lookups.
|
|
135
|
+
* **Maintenance:** stats, health, verify, checkpoint, compaction (in place or
|
|
136
|
+
as a copy), tail inspection and repair.
|
|
137
|
+
|
|
138
|
+
Measured throughput figures and their exact test conditions are documented
|
|
139
|
+
in the
|
|
140
|
+
[technical documentation, section 16.3](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/en/dNHyDB_Technical_Documentation.md#163-benchmarks).
|
|
141
|
+
|
|
142
|
+
## Documentation
|
|
143
|
+
|
|
144
|
+
* **[Python Cookbook](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/cookbook/PYTHON_COOKBOOK.md)** –
|
|
145
|
+
installation, the model, records and cells, addresses and `N`, types,
|
|
146
|
+
iterators, batches, NumPy, transactions and durability, readers and
|
|
147
|
+
writers, options, the VALUE index, maintenance, backups, errors, the
|
|
148
|
+
command line, complete application examples and an API reference. All
|
|
149
|
+
examples are executed by the test suite.
|
|
150
|
+
* [Technical documentation (English)](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/en/dNHyDB_Technical_Documentation.md) ·
|
|
151
|
+
[Technische Dokumentation (Deutsch)](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/de/dNHyDB_Technische_Dokumentation.md) –
|
|
152
|
+
mathematics, architecture, file format, guarantees.
|
|
153
|
+
* [Example programs](https://github.com/nhydb-run4/dNHyDB/tree/main/examples/python) ·
|
|
154
|
+
[example configuration](https://github.com/nhydb-run4/dNHyDB/blob/main/config/dnhydb_config.example.toml) ·
|
|
155
|
+
[`CHANGELOG.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/CHANGELOG.md)
|
|
156
|
+
* Command line: `dnhydb --help` (or `python -m dnhydb`).
|
|
157
|
+
* Rust: the core crate `crates/dnhydb-core` has no Python dependency and can
|
|
158
|
+
be used directly; see the technical documentation.
|
|
159
|
+
|
|
160
|
+
## Sibling project: NHyDB
|
|
161
|
+
|
|
162
|
+
[**NHyDB**](https://github.com/nhydb-run4/NHyDB) (`pip install nhydb`) is the full
|
|
163
|
+
Dynadron database: several realms, layouts CLASSIC, COLUMN, PACKED and
|
|
164
|
+
CUSTOM, MATH_MAP (placing values by formulas), text and full-text indexes,
|
|
165
|
+
the QueryEngine and NHySQL, and an encrypted external layout plan. Choose
|
|
166
|
+
**dNHyDB** for a compact, direct record store and **NHyDB** when you need
|
|
167
|
+
the full model.
|
|
168
|
+
|
|
169
|
+
## Scientific background
|
|
170
|
+
|
|
171
|
+
The mathematical foundations (in German):
|
|
172
|
+
|
|
173
|
+
* *Das Dynadron als mathematisches Datenbankprinzip* –
|
|
174
|
+
[doi:10.5281/zenodo.23438748](https://doi.org/10.5281/zenodo.23438748)
|
|
175
|
+
* *Das Dynadron der natürlichen Zahlen* –
|
|
176
|
+
[doi:10.5281/zenodo.22549415](https://doi.org/10.5281/zenodo.22549415)
|
|
177
|
+
* *Geometrie und Arithmetik im Dynadron* –
|
|
178
|
+
[doi:10.5281/zenodo.22694525](https://doi.org/10.5281/zenodo.22694525)
|
|
179
|
+
|
|
180
|
+
## Known limitations
|
|
181
|
+
|
|
182
|
+
* **Scope:** one realm (R1) and the CLASSIC record layout only; `u` and `v`
|
|
183
|
+
are `u64`; integer values up to 128 bits; no query language (access via
|
|
184
|
+
API, iterators and the VALUE index).
|
|
185
|
+
* **One writer at a time** (exclusive lock on `<file>.lock`); readers see the
|
|
186
|
+
last commit and call `refresh()` for newer commits.
|
|
187
|
+
* **`None` in BASE fields:** use extensions or free cells for optional values
|
|
188
|
+
(see the Cookbook).
|
|
189
|
+
* **Durability `full`:** on macOS `F_FULLFSYNC` is used; if the file system
|
|
190
|
+
rejects it, dNHyDB falls back to a plain `fsync` without reporting an
|
|
191
|
+
error. On Linux `full` is the same as `fsync`; on Windows both use
|
|
192
|
+
`FlushFileBuffers`.
|
|
193
|
+
* **Windows:** replacing the database file while readers hold it open
|
|
194
|
+
(compaction) uses POSIX rename semantics (Windows 10 or newer, NTFS).
|
|
195
|
+
* **Status:** alpha – the API may still change between minor versions.
|
|
196
|
+
|
|
197
|
+
## License
|
|
198
|
+
|
|
199
|
+
dNHyDB is licensed under the **NHyDB / dNHyDB Experimental Software License
|
|
200
|
+
1.0** ([`LICENSE.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/LICENSE.md)).
|
|
201
|
+
The source code is available, but this is **not an open-source license**:
|
|
202
|
+
free use is limited to non-production purposes (unlimited for private
|
|
203
|
+
individuals, time-limited for teaching, research and businesses), and
|
|
204
|
+
**production use is not permitted** without a separate written license.
|
|
205
|
+
Please read the license text before using the software.
|
|
206
|
+
|
|
207
|
+
The compiled extension module contains third-party Rust components under
|
|
208
|
+
MIT/Apache-2.0-style licenses; their notices are in
|
|
209
|
+
[`THIRD-PARTY-LICENSES.txt`](https://github.com/nhydb-run4/dNHyDB/blob/main/THIRD-PARTY-LICENSES.txt),
|
|
210
|
+
which is also shipped inside the wheel.
|
|
211
|
+
|
|
212
|
+
## Development
|
|
213
|
+
|
|
214
|
+
Building from source: [`BUILDING.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/BUILDING.md) ·
|
|
215
|
+
release procedure: [`RELEASING.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/RELEASING.md)
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
cargo clippy --workspace --all-targets -- -D warnings
|
|
219
|
+
pip install ".[test]"
|
|
220
|
+
python -m pytest tests/smoke # smoke tests of the installed package
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## Contact
|
|
224
|
+
|
|
225
|
+
Achim Meissner – achim.meissner@nhydb.de
|
|
226
|
+
|
dnhydb-0.3.0/README.md
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# dNHyDB (directNHyDB)
|
|
2
|
+
|
|
3
|
+
**A compact record store with direct mathematical addressing.**
|
|
4
|
+
|
|
5
|
+
dNHyDB is the streamlined member of the **Dynadron** database family. Every
|
|
6
|
+
value lives in a cell `(u, v, s)`: `u` is the record, `v` the field or
|
|
7
|
+
position, `s` one of two sides. There are no tables and no primary index –
|
|
8
|
+
the address space itself provides order, and each cell has an exact natural
|
|
9
|
+
number `N` that is computed when needed, never stored. The engine is written
|
|
10
|
+
in native Rust and used from Python.
|
|
11
|
+
|
|
12
|
+
[](https://pypi.org/project/dnhydb/)
|
|
13
|
+
· Python 3.12–3.15 · Linux, macOS, Windows ·
|
|
14
|
+
[License: source-available, non-production](#license)
|
|
15
|
+
|
|
16
|
+
**→ [Python Cookbook](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/cookbook/PYTHON_COOKBOOK.md)** –
|
|
17
|
+
task-oriented guide with tested examples for the complete Python API.
|
|
18
|
+
|
|
19
|
+
## The addressing idea
|
|
20
|
+
|
|
21
|
+
`u` and `v` are integers from 1 to 2⁶⁴ − 1, the side `s` is `NL` or `NR`.
|
|
22
|
+
The number of a cell is
|
|
23
|
+
|
|
24
|
+
```text
|
|
25
|
+
G = u + v N = G² − u (side NL) N = G² + v (side NR)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
so `(u, v) = (11, 13)` is `N = 565` on NL and `N = 589` on NR. The mapping is
|
|
29
|
+
exact in both directions (`dnhydb.n_of`, `dnhydb.address_of`); `N` needs up
|
|
30
|
+
to 130 bits and is handled as a Python integer.
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install dnhydb
|
|
36
|
+
pip install numpy # optional, for the NumPy batch API
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Prebuilt wheels are available for CPython 3.12–3.15 on Linux (x86_64,
|
|
40
|
+
aarch64), macOS (x86_64, arm64) and Windows (x64). No Rust toolchain is
|
|
41
|
+
needed; building from source is described in
|
|
42
|
+
[`BUILDING.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/BUILDING.md).
|
|
43
|
+
|
|
44
|
+
## A first example
|
|
45
|
+
|
|
46
|
+
Create a database, write two records and a free cell, close it, open it
|
|
47
|
+
again as a reader and read:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import os
|
|
51
|
+
import tempfile
|
|
52
|
+
|
|
53
|
+
import dnhydb
|
|
54
|
+
|
|
55
|
+
path = os.path.join(tempfile.mkdtemp(), "cities.dnhydb")
|
|
56
|
+
|
|
57
|
+
# Create: a typed BASE record layout, two records and one free cell.
|
|
58
|
+
with dnhydb.create(path) as db:
|
|
59
|
+
db.define_base([("city", "utf8"), ("population", "u64")])
|
|
60
|
+
db.put_record(1, ["Berlin", 3_685_000])
|
|
61
|
+
db.put_record(2, ["Hamburg", 1_862_000])
|
|
62
|
+
db.put_cell(2, 1_000, "port city")
|
|
63
|
+
# Leaving the block commits and closes the database.
|
|
64
|
+
|
|
65
|
+
db = dnhydb.open_reader(path)
|
|
66
|
+
print(db.get_record(2))
|
|
67
|
+
print(db.get_cell(2, 1_000))
|
|
68
|
+
n = dnhydb.n_of(2, 1)
|
|
69
|
+
print(n, dnhydb.address_of(n))
|
|
70
|
+
db.close()
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
```text
|
|
74
|
+
{'city': 'Hamburg', 'population': 1862000, 'v1000': 'port city'}
|
|
75
|
+
port city
|
|
76
|
+
7 (2, 1, 0)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The database is a single file (`cities.dnhydb`); next to it the engine keeps
|
|
80
|
+
a `.lock` file for the writer and a catalog (`.dnhydb-index`) that speeds up
|
|
81
|
+
opening and is rebuilt automatically. For a backup the `.dnhydb` file is
|
|
82
|
+
enough – see
|
|
83
|
+
[Cookbook, section 17](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/cookbook/PYTHON_COOKBOOK.md#17-files-on-disk-backup-and-restore).
|
|
84
|
+
|
|
85
|
+
## What dNHyDB offers
|
|
86
|
+
|
|
87
|
+
* **Records and cells:** a typed BASE record layout (`define_base`), named
|
|
88
|
+
extensions, free cells anywhere in the address space, deletion with
|
|
89
|
+
tombstones, two sides per cell.
|
|
90
|
+
* **Write path:** write buffer, batches, NumPy column batches, commit policy
|
|
91
|
+
(manual or by thresholds), transactions with rollback, durability `fsync`,
|
|
92
|
+
`full` or `none`.
|
|
93
|
+
* **One writer, many readers:** a reader – also in another process – sees
|
|
94
|
+
the last commit as a consistent snapshot and moves forward with
|
|
95
|
+
`refresh()`.
|
|
96
|
+
* **Crash safety:** append-only file with two CRC-protected superblocks; after
|
|
97
|
+
a crash the last commit is valid, never half a batch.
|
|
98
|
+
* **Reading:** point access, records, iterators that skip gaps, column reads
|
|
99
|
+
into NumPy, a VALUE index for equality and range lookups.
|
|
100
|
+
* **Maintenance:** stats, health, verify, checkpoint, compaction (in place or
|
|
101
|
+
as a copy), tail inspection and repair.
|
|
102
|
+
|
|
103
|
+
Measured throughput figures and their exact test conditions are documented
|
|
104
|
+
in the
|
|
105
|
+
[technical documentation, section 16.3](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/en/dNHyDB_Technical_Documentation.md#163-benchmarks).
|
|
106
|
+
|
|
107
|
+
## Documentation
|
|
108
|
+
|
|
109
|
+
* **[Python Cookbook](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/cookbook/PYTHON_COOKBOOK.md)** –
|
|
110
|
+
installation, the model, records and cells, addresses and `N`, types,
|
|
111
|
+
iterators, batches, NumPy, transactions and durability, readers and
|
|
112
|
+
writers, options, the VALUE index, maintenance, backups, errors, the
|
|
113
|
+
command line, complete application examples and an API reference. All
|
|
114
|
+
examples are executed by the test suite.
|
|
115
|
+
* [Technical documentation (English)](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/en/dNHyDB_Technical_Documentation.md) ·
|
|
116
|
+
[Technische Dokumentation (Deutsch)](https://github.com/nhydb-run4/dNHyDB/blob/main/docs/de/dNHyDB_Technische_Dokumentation.md) –
|
|
117
|
+
mathematics, architecture, file format, guarantees.
|
|
118
|
+
* [Example programs](https://github.com/nhydb-run4/dNHyDB/tree/main/examples/python) ·
|
|
119
|
+
[example configuration](https://github.com/nhydb-run4/dNHyDB/blob/main/config/dnhydb_config.example.toml) ·
|
|
120
|
+
[`CHANGELOG.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/CHANGELOG.md)
|
|
121
|
+
* Command line: `dnhydb --help` (or `python -m dnhydb`).
|
|
122
|
+
* Rust: the core crate `crates/dnhydb-core` has no Python dependency and can
|
|
123
|
+
be used directly; see the technical documentation.
|
|
124
|
+
|
|
125
|
+
## Sibling project: NHyDB
|
|
126
|
+
|
|
127
|
+
[**NHyDB**](https://github.com/nhydb-run4/NHyDB) (`pip install nhydb`) is the full
|
|
128
|
+
Dynadron database: several realms, layouts CLASSIC, COLUMN, PACKED and
|
|
129
|
+
CUSTOM, MATH_MAP (placing values by formulas), text and full-text indexes,
|
|
130
|
+
the QueryEngine and NHySQL, and an encrypted external layout plan. Choose
|
|
131
|
+
**dNHyDB** for a compact, direct record store and **NHyDB** when you need
|
|
132
|
+
the full model.
|
|
133
|
+
|
|
134
|
+
## Scientific background
|
|
135
|
+
|
|
136
|
+
The mathematical foundations (in German):
|
|
137
|
+
|
|
138
|
+
* *Das Dynadron als mathematisches Datenbankprinzip* –
|
|
139
|
+
[doi:10.5281/zenodo.23438748](https://doi.org/10.5281/zenodo.23438748)
|
|
140
|
+
* *Das Dynadron der natürlichen Zahlen* –
|
|
141
|
+
[doi:10.5281/zenodo.22549415](https://doi.org/10.5281/zenodo.22549415)
|
|
142
|
+
* *Geometrie und Arithmetik im Dynadron* –
|
|
143
|
+
[doi:10.5281/zenodo.22694525](https://doi.org/10.5281/zenodo.22694525)
|
|
144
|
+
|
|
145
|
+
## Known limitations
|
|
146
|
+
|
|
147
|
+
* **Scope:** one realm (R1) and the CLASSIC record layout only; `u` and `v`
|
|
148
|
+
are `u64`; integer values up to 128 bits; no query language (access via
|
|
149
|
+
API, iterators and the VALUE index).
|
|
150
|
+
* **One writer at a time** (exclusive lock on `<file>.lock`); readers see the
|
|
151
|
+
last commit and call `refresh()` for newer commits.
|
|
152
|
+
* **`None` in BASE fields:** use extensions or free cells for optional values
|
|
153
|
+
(see the Cookbook).
|
|
154
|
+
* **Durability `full`:** on macOS `F_FULLFSYNC` is used; if the file system
|
|
155
|
+
rejects it, dNHyDB falls back to a plain `fsync` without reporting an
|
|
156
|
+
error. On Linux `full` is the same as `fsync`; on Windows both use
|
|
157
|
+
`FlushFileBuffers`.
|
|
158
|
+
* **Windows:** replacing the database file while readers hold it open
|
|
159
|
+
(compaction) uses POSIX rename semantics (Windows 10 or newer, NTFS).
|
|
160
|
+
* **Status:** alpha – the API may still change between minor versions.
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
dNHyDB is licensed under the **NHyDB / dNHyDB Experimental Software License
|
|
165
|
+
1.0** ([`LICENSE.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/LICENSE.md)).
|
|
166
|
+
The source code is available, but this is **not an open-source license**:
|
|
167
|
+
free use is limited to non-production purposes (unlimited for private
|
|
168
|
+
individuals, time-limited for teaching, research and businesses), and
|
|
169
|
+
**production use is not permitted** without a separate written license.
|
|
170
|
+
Please read the license text before using the software.
|
|
171
|
+
|
|
172
|
+
The compiled extension module contains third-party Rust components under
|
|
173
|
+
MIT/Apache-2.0-style licenses; their notices are in
|
|
174
|
+
[`THIRD-PARTY-LICENSES.txt`](https://github.com/nhydb-run4/dNHyDB/blob/main/THIRD-PARTY-LICENSES.txt),
|
|
175
|
+
which is also shipped inside the wheel.
|
|
176
|
+
|
|
177
|
+
## Development
|
|
178
|
+
|
|
179
|
+
Building from source: [`BUILDING.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/BUILDING.md) ·
|
|
180
|
+
release procedure: [`RELEASING.md`](https://github.com/nhydb-run4/dNHyDB/blob/main/RELEASING.md)
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
cargo clippy --workspace --all-targets -- -D warnings
|
|
184
|
+
pip install ".[test]"
|
|
185
|
+
python -m pytest tests/smoke # smoke tests of the installed package
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
## Contact
|
|
189
|
+
|
|
190
|
+
Achim Meissner – achim.meissner@nhydb.de
|
|
@@ -19,3 +19,7 @@ libc.workspace = true
|
|
|
19
19
|
|
|
20
20
|
[target.'cfg(windows)'.dependencies]
|
|
21
21
|
windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem", "Win32_Foundation", "Win32_System_SystemInformation"] }
|
|
22
|
+
|
|
23
|
+
[lints.rust]
|
|
24
|
+
# unit tests that are not part of the published sources
|
|
25
|
+
unexpected_cfgs = { level = "warn", check-cfg = ["cfg(dnhydb_internal_tests)"] }
|
|
@@ -186,95 +186,6 @@ pub fn dyn_address(u: u64, v: u64) -> Result<DynAddress> {
|
|
|
186
186
|
Ok(DynAddress { u, v, g, station, stufenbreite, stufe, stufenstation, nl, nr })
|
|
187
187
|
}
|
|
188
188
|
|
|
189
|
-
#[cfg(test)]
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
fn n(x: u128) -> U256 {
|
|
194
|
-
U256::from_u128(x)
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
#[test]
|
|
198
|
-
fn examples_from_docs() {
|
|
199
|
-
let a = address_of(n(589)).unwrap();
|
|
200
|
-
assert_eq!((a.u, a.v, a.lane), (11, 13, Lane::NR));
|
|
201
|
-
assert_eq!(n_of(11, 13, Lane::NL).unwrap(), n(565));
|
|
202
|
-
let b = address_of(n(391)).unwrap();
|
|
203
|
-
assert_eq!((b.u, b.v, b.lane), (9, 11, Lane::NL));
|
|
204
|
-
for bad in [552u128, 576, 600, 4] {
|
|
205
|
-
assert!(matches!(address_of(n(bad)), Err(Error::Address(_))), "{bad}");
|
|
206
|
-
}
|
|
207
|
-
for (good, lane) in [(553u128, Lane::NL), (575, Lane::NL), (577, Lane::NR), (599, Lane::NR)] {
|
|
208
|
-
let a = address_of(n(good)).unwrap();
|
|
209
|
-
assert_eq!(a.lane, lane);
|
|
210
|
-
assert_eq!(a.u as u128 + a.v as u128, 24);
|
|
211
|
-
}
|
|
212
|
-
assert_eq!(n_of(1, 1, Lane::NL).unwrap(), n(3));
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
#[test]
|
|
216
|
-
fn boundaries_roundtrip() {
|
|
217
|
-
let edges = [1u64, 2, 1 << 32, (1 << 32) + 1, 1 << 63, u64::MAX - 1, u64::MAX];
|
|
218
|
-
for &u in &edges {
|
|
219
|
-
for &v in &edges {
|
|
220
|
-
for lane in LANES {
|
|
221
|
-
let a = CellAddress::new(u, v, lane).unwrap();
|
|
222
|
-
assert!(a.n.bits() <= N_PROFILE_BITS);
|
|
223
|
-
let b = address_of(a.n).unwrap();
|
|
224
|
-
assert_eq!(a, b, "u={u} v={v} {lane:?}");
|
|
225
|
-
assert_eq!(parse_n(&a.n_ref()).unwrap(), a.n);
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
// Maximum: u = v = 2^64-1
|
|
230
|
-
let (g, _, nr) = n_pair(u64::MAX, u64::MAX);
|
|
231
|
-
assert_eq!(g, (1u128 << 65) - 2);
|
|
232
|
-
assert_eq!(nr.to_string(), "1361129467683753853724371221211105984515");
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
#[test]
|
|
236
|
-
fn invalid_inputs() {
|
|
237
|
-
assert!(matches!(n_of(0, 1, Lane::NL), Err(Error::Address(_))));
|
|
238
|
-
assert!(matches!(n_of(1, 0, Lane::NR), Err(Error::Address(_))));
|
|
239
|
-
assert!(matches!(address_of(U256::ZERO), Err(Error::Address(_))));
|
|
240
|
-
assert!(matches!(address_of(U256 { hi: 1 << 10, lo: 0 }), Err(Error::Profile(_))));
|
|
241
|
-
// N with v = 2^64 (outside): G = 2^64 + 1, u = 1, NR
|
|
242
|
-
let g = (1u128 << 64) + 1;
|
|
243
|
-
let nr = U256::square_u128(g).add_u128(1 << 64).unwrap();
|
|
244
|
-
assert!(matches!(address_of(nr), Err(Error::Profile(_))));
|
|
245
|
-
assert!(matches!(parse_n("Nx"), Err(Error::Address(_))));
|
|
246
|
-
assert!(matches!(parse_n("0"), Err(Error::Address(_))));
|
|
247
|
-
assert_eq!(parse_n(" n 12 ").unwrap(), n(12));
|
|
248
|
-
assert_eq!(Lane::parse("nr").unwrap(), Lane::NR);
|
|
249
|
-
assert!(Lane::from_int(2).is_err());
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
#[test]
|
|
253
|
-
fn exhaustive_small_range() {
|
|
254
|
-
let mut valid = 0;
|
|
255
|
-
for x in 1u128..200_000 {
|
|
256
|
-
let m = crate::u256::isqrt_u128(x);
|
|
257
|
-
let is_gz = m * m == x;
|
|
258
|
-
let is_n0 = m * (m + 1) == x;
|
|
259
|
-
match address_of(n(x)) {
|
|
260
|
-
Ok(a) => {
|
|
261
|
-
assert!(!is_gz && !is_n0);
|
|
262
|
-
assert_eq!(n_of(a.u, a.v, a.lane).unwrap(), n(x));
|
|
263
|
-
valid += 1;
|
|
264
|
-
}
|
|
265
|
-
Err(_) => assert!(is_gz || is_n0, "{x}"),
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
assert!(valid > 199_000);
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
#[test]
|
|
272
|
-
fn dyn_values() {
|
|
273
|
-
let d = dyn_address(11, 13).unwrap();
|
|
274
|
-
assert_eq!((d.g, d.station, d.stufenbreite, d.stufe, d.stufenstation), (24, 12, 13, 1, 12));
|
|
275
|
-
assert_eq!(d.nl, n(565));
|
|
276
|
-
assert_eq!(d.nr, n(589));
|
|
277
|
-
let d = dyn_address(u64::MAX, u64::MAX).unwrap();
|
|
278
|
-
assert_eq!(d.stufe, 1);
|
|
279
|
-
}
|
|
280
|
-
}
|
|
189
|
+
#[cfg(all(test, dnhydb_internal_tests))]
|
|
190
|
+
#[path = "address_tests.rs"]
|
|
191
|
+
mod tests;
|