triepack 1.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- triepack-1.3.1/LICENSE.txt +23 -0
- triepack-1.3.1/MANIFEST.in +4 -0
- triepack-1.3.1/PKG-INFO +107 -0
- triepack-1.3.1/README.md +73 -0
- triepack-1.3.1/pyproject.toml +59 -0
- triepack-1.3.1/setup.cfg +4 -0
- triepack-1.3.1/setup.py +5 -0
- triepack-1.3.1/triepack/__init__.py +44 -0
- triepack-1.3.1/triepack/bitstream.py +160 -0
- triepack-1.3.1/triepack/crc32.py +47 -0
- triepack-1.3.1/triepack/decoder.py +182 -0
- triepack-1.3.1/triepack/encoder.py +346 -0
- triepack-1.3.1/triepack/values.py +96 -0
- triepack-1.3.1/triepack/varint.py +59 -0
- triepack-1.3.1/triepack.egg-info/PKG-INFO +107 -0
- triepack-1.3.1/triepack.egg-info/SOURCES.txt +17 -0
- triepack-1.3.1/triepack.egg-info/dependency_links.txt +1 -0
- triepack-1.3.1/triepack.egg-info/requires.txt +3 -0
- triepack-1.3.1/triepack.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
triepack is Copyright (c) 2026, M. A. Chatterjee <deftio at deftio dot com>
|
|
2
|
+
All rights reserved.
|
|
3
|
+
|
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
|
5
|
+
modification, are permitted provided that the following conditions are met:
|
|
6
|
+
|
|
7
|
+
* Redistributions of source code must retain the above copyright notice, this
|
|
8
|
+
list of conditions and the following disclaimer.
|
|
9
|
+
|
|
10
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
|
11
|
+
this list of conditions and the following disclaimer in the documentation
|
|
12
|
+
and/or other materials provided with the distribution.
|
|
13
|
+
|
|
14
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
15
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
16
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
17
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
18
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
19
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
20
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
21
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
22
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
23
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
triepack-1.3.1/PKG-INFO
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: triepack
|
|
3
|
+
Version: 1.3.1
|
|
4
|
+
Summary: Compressed trie dictionary format (.trp) — compact binary key-value storage with fast lookups and prefix search. Pure Python, no dependencies.
|
|
5
|
+
Author-email: "M. A. Chatterjee" <deftio@deftio.com>
|
|
6
|
+
License-Expression: BSD-2-Clause
|
|
7
|
+
Project-URL: Homepage, https://deftio.github.io/triepack/
|
|
8
|
+
Project-URL: Documentation, https://deftio.github.io/triepack/guide/api-reference/
|
|
9
|
+
Project-URL: Repository, https://github.com/deftio/triepack
|
|
10
|
+
Project-URL: Issues, https://github.com/deftio/triepack/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/deftio/triepack/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: trie,prefix-tree,dictionary,binary-format,serialization,compression,key-value,prefix-search,trp,triepack
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
24
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Classifier: Topic :: System :: Archiving :: Compression
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.8
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENSE.txt
|
|
31
|
+
Provides-Extra: test
|
|
32
|
+
Requires-Dist: pytest; extra == "test"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# triepack
|
|
36
|
+
|
|
37
|
+
Compressed trie dictionary format (`.trp`) — compact binary key-value storage
|
|
38
|
+
with fast lookups and prefix search.
|
|
39
|
+
|
|
40
|
+
This is the native Python implementation: pure Python, no dependencies, no C
|
|
41
|
+
extension to build. It reads and writes the same bytes as the C reference
|
|
42
|
+
library and the JavaScript, Go, Rust, Swift, Java and Kotlin implementations.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install triepack
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Use
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from triepack import encode, decode
|
|
54
|
+
|
|
55
|
+
buf = encode({"hello": 42, "world": "foo"})
|
|
56
|
+
# buf is bytes holding the .trp binary
|
|
57
|
+
|
|
58
|
+
result = decode(buf)
|
|
59
|
+
print(result) # {'hello': 42, 'world': 'foo'}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Values
|
|
63
|
+
|
|
64
|
+
| Python | `.trp` type |
|
|
65
|
+
|-------------------|--------------------|
|
|
66
|
+
| `None` | null |
|
|
67
|
+
| `bool` | bool |
|
|
68
|
+
| `int >= 0` | uint |
|
|
69
|
+
| `int < 0` | int |
|
|
70
|
+
| `float` | float64 |
|
|
71
|
+
| `str` | string (UTF-8) |
|
|
72
|
+
| `bytes` | blob |
|
|
73
|
+
|
|
74
|
+
Decoding also accepts float32 values written by other implementations,
|
|
75
|
+
widening them to a Python float. Python integers are arbitrary precision, so
|
|
76
|
+
the full 64-bit signed and unsigned ranges round-trip exactly.
|
|
77
|
+
|
|
78
|
+
## Format
|
|
79
|
+
|
|
80
|
+
Every buffer carries a 32-byte header, a bit-packed prefix trie, a typed value
|
|
81
|
+
store and a CRC-32. Keys are stored once per shared prefix, and symbols are
|
|
82
|
+
packed at the minimum width the key alphabet needs.
|
|
83
|
+
|
|
84
|
+
The encoder is deterministic: the same input produces the same bytes in every
|
|
85
|
+
implementation, which is checked by a
|
|
86
|
+
[shared conformance suite](https://github.com/deftio/triepack/tree/main/tests/conformance)
|
|
87
|
+
that all nine implementations run.
|
|
88
|
+
|
|
89
|
+
## Links
|
|
90
|
+
|
|
91
|
+
- [Documentation](https://deftio.github.io/triepack/)
|
|
92
|
+
- [API reference](https://deftio.github.io/triepack/guide/api-reference/)
|
|
93
|
+
- [Source and issues](https://github.com/deftio/triepack)
|
|
94
|
+
- [Changelog](https://github.com/deftio/triepack/blob/main/CHANGELOG.md)
|
|
95
|
+
|
|
96
|
+
## Development
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
cd bindings/python
|
|
100
|
+
pip install -e ".[test]"
|
|
101
|
+
python -m pytest
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## License
|
|
105
|
+
|
|
106
|
+
Copyright (c) 2026 M. A. Chatterjee. BSD-2-Clause — see
|
|
107
|
+
[LICENSE.txt](LICENSE.txt).
|
triepack-1.3.1/README.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# triepack
|
|
2
|
+
|
|
3
|
+
Compressed trie dictionary format (`.trp`) — compact binary key-value storage
|
|
4
|
+
with fast lookups and prefix search.
|
|
5
|
+
|
|
6
|
+
This is the native Python implementation: pure Python, no dependencies, no C
|
|
7
|
+
extension to build. It reads and writes the same bytes as the C reference
|
|
8
|
+
library and the JavaScript, Go, Rust, Swift, Java and Kotlin implementations.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install triepack
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Use
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
from triepack import encode, decode
|
|
20
|
+
|
|
21
|
+
buf = encode({"hello": 42, "world": "foo"})
|
|
22
|
+
# buf is bytes holding the .trp binary
|
|
23
|
+
|
|
24
|
+
result = decode(buf)
|
|
25
|
+
print(result) # {'hello': 42, 'world': 'foo'}
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Values
|
|
29
|
+
|
|
30
|
+
| Python | `.trp` type |
|
|
31
|
+
|-------------------|--------------------|
|
|
32
|
+
| `None` | null |
|
|
33
|
+
| `bool` | bool |
|
|
34
|
+
| `int >= 0` | uint |
|
|
35
|
+
| `int < 0` | int |
|
|
36
|
+
| `float` | float64 |
|
|
37
|
+
| `str` | string (UTF-8) |
|
|
38
|
+
| `bytes` | blob |
|
|
39
|
+
|
|
40
|
+
Decoding also accepts float32 values written by other implementations,
|
|
41
|
+
widening them to a Python float. Python integers are arbitrary precision, so
|
|
42
|
+
the full 64-bit signed and unsigned ranges round-trip exactly.
|
|
43
|
+
|
|
44
|
+
## Format
|
|
45
|
+
|
|
46
|
+
Every buffer carries a 32-byte header, a bit-packed prefix trie, a typed value
|
|
47
|
+
store and a CRC-32. Keys are stored once per shared prefix, and symbols are
|
|
48
|
+
packed at the minimum width the key alphabet needs.
|
|
49
|
+
|
|
50
|
+
The encoder is deterministic: the same input produces the same bytes in every
|
|
51
|
+
implementation, which is checked by a
|
|
52
|
+
[shared conformance suite](https://github.com/deftio/triepack/tree/main/tests/conformance)
|
|
53
|
+
that all nine implementations run.
|
|
54
|
+
|
|
55
|
+
## Links
|
|
56
|
+
|
|
57
|
+
- [Documentation](https://deftio.github.io/triepack/)
|
|
58
|
+
- [API reference](https://deftio.github.io/triepack/guide/api-reference/)
|
|
59
|
+
- [Source and issues](https://github.com/deftio/triepack)
|
|
60
|
+
- [Changelog](https://github.com/deftio/triepack/blob/main/CHANGELOG.md)
|
|
61
|
+
|
|
62
|
+
## Development
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
cd bindings/python
|
|
66
|
+
pip install -e ".[test]"
|
|
67
|
+
python -m pytest
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## License
|
|
71
|
+
|
|
72
|
+
Copyright (c) 2026 M. A. Chatterjee. BSD-2-Clause — see
|
|
73
|
+
[LICENSE.txt](LICENSE.txt).
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
|
|
3
|
+
[build-system]
|
|
4
|
+
requires = ["setuptools>=77"]
|
|
5
|
+
build-backend = "setuptools.build_meta"
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "triepack"
|
|
9
|
+
version = "1.3.1"
|
|
10
|
+
description = "Compressed trie dictionary format (.trp) — compact binary key-value storage with fast lookups and prefix search. Pure Python, no dependencies."
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
requires-python = ">=3.8"
|
|
13
|
+
license = "BSD-2-Clause"
|
|
14
|
+
license-files = ["LICENSE.txt"]
|
|
15
|
+
authors = [{ name = "M. A. Chatterjee", email = "deftio@deftio.com" }]
|
|
16
|
+
keywords = [
|
|
17
|
+
"trie",
|
|
18
|
+
"prefix-tree",
|
|
19
|
+
"dictionary",
|
|
20
|
+
"binary-format",
|
|
21
|
+
"serialization",
|
|
22
|
+
"compression",
|
|
23
|
+
"key-value",
|
|
24
|
+
"prefix-search",
|
|
25
|
+
"trp",
|
|
26
|
+
"triepack",
|
|
27
|
+
]
|
|
28
|
+
classifiers = [
|
|
29
|
+
"Development Status :: 5 - Production/Stable",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
"Operating System :: OS Independent",
|
|
32
|
+
"Programming Language :: Python :: 3",
|
|
33
|
+
"Programming Language :: Python :: 3.8",
|
|
34
|
+
"Programming Language :: Python :: 3.9",
|
|
35
|
+
"Programming Language :: Python :: 3.10",
|
|
36
|
+
"Programming Language :: Python :: 3.11",
|
|
37
|
+
"Programming Language :: Python :: 3.12",
|
|
38
|
+
"Programming Language :: Python :: 3.13",
|
|
39
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
40
|
+
"Programming Language :: Python :: Implementation :: PyPy",
|
|
41
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
42
|
+
"Topic :: System :: Archiving :: Compression",
|
|
43
|
+
"Typing :: Typed",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.urls]
|
|
47
|
+
Homepage = "https://deftio.github.io/triepack/"
|
|
48
|
+
Documentation = "https://deftio.github.io/triepack/guide/api-reference/"
|
|
49
|
+
Repository = "https://github.com/deftio/triepack"
|
|
50
|
+
Issues = "https://github.com/deftio/triepack/issues"
|
|
51
|
+
Changelog = "https://github.com/deftio/triepack/blob/main/CHANGELOG.md"
|
|
52
|
+
|
|
53
|
+
[project.optional-dependencies]
|
|
54
|
+
test = ["pytest"]
|
|
55
|
+
|
|
56
|
+
[tool.setuptools]
|
|
57
|
+
# Ship the package only. The test suite reads fixtures from tests/conformance/
|
|
58
|
+
# in the repository, which is not part of a release.
|
|
59
|
+
packages = ["triepack"]
|
triepack-1.3.1/setup.cfg
ADDED
triepack-1.3.1/setup.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
|
|
3
|
+
"""triepack — Native Python implementation of the Triepack .trp binary format."""
|
|
4
|
+
|
|
5
|
+
__version__ = "1.3.1"
|
|
6
|
+
|
|
7
|
+
from .decoder import decode
|
|
8
|
+
from .encoder import MAX_ALPHABET_SIZE, encode
|
|
9
|
+
|
|
10
|
+
#: Version of the on-disk .trp format this implementation writes. Distinct
|
|
11
|
+
#: from the library version: it changes only when the bytes change.
|
|
12
|
+
FORMAT_VERSION_MAJOR = 1
|
|
13
|
+
FORMAT_VERSION_MINOR = 0
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def version():
|
|
17
|
+
"""Return metadata about this build.
|
|
18
|
+
|
|
19
|
+
Every triepack implementation answers with the same shape, so a polyglot
|
|
20
|
+
system can ask each one what it is.
|
|
21
|
+
"""
|
|
22
|
+
major, minor, patch = (int(p) for p in __version__.split("."))
|
|
23
|
+
return {
|
|
24
|
+
"name": "triepack",
|
|
25
|
+
"implementation": "python",
|
|
26
|
+
"version": __version__,
|
|
27
|
+
"version_major": major,
|
|
28
|
+
"version_minor": minor,
|
|
29
|
+
"version_patch": patch,
|
|
30
|
+
"format_version_major": FORMAT_VERSION_MAJOR,
|
|
31
|
+
"format_version_minor": FORMAT_VERSION_MINOR,
|
|
32
|
+
"max_alphabet_size": MAX_ALPHABET_SIZE,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"encode",
|
|
38
|
+
"decode",
|
|
39
|
+
"version",
|
|
40
|
+
"MAX_ALPHABET_SIZE",
|
|
41
|
+
"FORMAT_VERSION_MAJOR",
|
|
42
|
+
"FORMAT_VERSION_MINOR",
|
|
43
|
+
"__version__",
|
|
44
|
+
]
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
"""Arbitrary-width bit I/O — MSB-first, matching src/bitstream/."""
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class BitWriter:
|
|
6
|
+
"""Write arbitrary-width bit fields into a growable buffer."""
|
|
7
|
+
|
|
8
|
+
def __init__(self, initial_cap=256):
|
|
9
|
+
self._buf = bytearray(initial_cap or 256)
|
|
10
|
+
self._pos = 0 # bit position
|
|
11
|
+
|
|
12
|
+
@property
|
|
13
|
+
def position(self):
|
|
14
|
+
return self._pos
|
|
15
|
+
|
|
16
|
+
def _ensure(self, n_bits):
|
|
17
|
+
needed = (self._pos + n_bits + 7) >> 3
|
|
18
|
+
if needed <= len(self._buf):
|
|
19
|
+
return
|
|
20
|
+
new_cap = len(self._buf) * 2
|
|
21
|
+
while new_cap < needed:
|
|
22
|
+
new_cap *= 2
|
|
23
|
+
self._buf.extend(b"\x00" * (new_cap - len(self._buf)))
|
|
24
|
+
|
|
25
|
+
def write_bits(self, value, n):
|
|
26
|
+
if n == 0 or n > 64:
|
|
27
|
+
raise ValueError("write_bits: n must be 1..64")
|
|
28
|
+
self._ensure(n)
|
|
29
|
+
for i in range(n):
|
|
30
|
+
bit = (value >> (n - 1 - i)) & 1
|
|
31
|
+
byte_idx = self._pos >> 3
|
|
32
|
+
bit_idx = 7 - (self._pos & 7)
|
|
33
|
+
if bit:
|
|
34
|
+
self._buf[byte_idx] |= 1 << bit_idx
|
|
35
|
+
else:
|
|
36
|
+
self._buf[byte_idx] &= ~(1 << bit_idx)
|
|
37
|
+
self._pos += 1
|
|
38
|
+
|
|
39
|
+
def write_bit(self, val):
|
|
40
|
+
self._ensure(1)
|
|
41
|
+
byte_idx = self._pos >> 3
|
|
42
|
+
bit_idx = 7 - (self._pos & 7)
|
|
43
|
+
if val:
|
|
44
|
+
self._buf[byte_idx] |= 1 << bit_idx
|
|
45
|
+
else:
|
|
46
|
+
self._buf[byte_idx] &= ~(1 << bit_idx)
|
|
47
|
+
self._pos += 1
|
|
48
|
+
|
|
49
|
+
def write_u8(self, val):
|
|
50
|
+
self.write_bits(val & 0xFF, 8)
|
|
51
|
+
|
|
52
|
+
def write_u16(self, val):
|
|
53
|
+
self.write_bits(val & 0xFFFF, 16)
|
|
54
|
+
|
|
55
|
+
def write_u32(self, val):
|
|
56
|
+
self.write_bits(val & 0xFFFFFFFF, 32)
|
|
57
|
+
|
|
58
|
+
def write_u64(self, hi32, lo32):
|
|
59
|
+
self.write_bits((hi32 >> 0) & 0xFFFFFFFF, 32)
|
|
60
|
+
self.write_bits((lo32 >> 0) & 0xFFFFFFFF, 32)
|
|
61
|
+
|
|
62
|
+
def write_bytes(self, data):
|
|
63
|
+
for b in data:
|
|
64
|
+
self.write_u8(b)
|
|
65
|
+
|
|
66
|
+
def align_to_byte(self):
|
|
67
|
+
rem = self._pos & 7
|
|
68
|
+
if rem != 0:
|
|
69
|
+
pad = 8 - rem
|
|
70
|
+
self._ensure(pad)
|
|
71
|
+
self._pos += pad
|
|
72
|
+
|
|
73
|
+
def to_bytes(self):
|
|
74
|
+
length = (self._pos + 7) >> 3
|
|
75
|
+
return bytes(self._buf[:length])
|
|
76
|
+
|
|
77
|
+
def get_buffer(self):
|
|
78
|
+
return self._buf
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class BitReader:
|
|
82
|
+
"""Read arbitrary-width bit fields from a buffer."""
|
|
83
|
+
|
|
84
|
+
def __init__(self, buf):
|
|
85
|
+
if isinstance(buf, (bytes, bytearray)):
|
|
86
|
+
self._buf = buf
|
|
87
|
+
else:
|
|
88
|
+
self._buf = bytes(buf)
|
|
89
|
+
self._bit_len = len(self._buf) * 8
|
|
90
|
+
self._pos = 0
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def position(self):
|
|
94
|
+
return self._pos
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def remaining(self):
|
|
98
|
+
return self._bit_len - self._pos
|
|
99
|
+
|
|
100
|
+
def seek(self, bit_pos):
|
|
101
|
+
self._pos = bit_pos
|
|
102
|
+
|
|
103
|
+
def advance(self, n_bits):
|
|
104
|
+
self._pos += n_bits
|
|
105
|
+
|
|
106
|
+
def read_bits(self, n):
|
|
107
|
+
if n == 0 or n > 64:
|
|
108
|
+
raise ValueError("read_bits: n must be 1..64")
|
|
109
|
+
if self._pos + n > self._bit_len:
|
|
110
|
+
raise EOFError("EOF")
|
|
111
|
+
val = 0
|
|
112
|
+
for i in range(n):
|
|
113
|
+
byte_idx = self._pos >> 3
|
|
114
|
+
bit_idx = 7 - (self._pos & 7)
|
|
115
|
+
val = (val << 1) | ((self._buf[byte_idx] >> bit_idx) & 1)
|
|
116
|
+
self._pos += 1
|
|
117
|
+
return val
|
|
118
|
+
|
|
119
|
+
def read_bit(self):
|
|
120
|
+
if self._pos >= self._bit_len:
|
|
121
|
+
raise EOFError("EOF")
|
|
122
|
+
byte_idx = self._pos >> 3
|
|
123
|
+
bit_idx = 7 - (self._pos & 7)
|
|
124
|
+
bit = (self._buf[byte_idx] >> bit_idx) & 1
|
|
125
|
+
self._pos += 1
|
|
126
|
+
return bit
|
|
127
|
+
|
|
128
|
+
def read_u8(self):
|
|
129
|
+
return self.read_bits(8)
|
|
130
|
+
|
|
131
|
+
def read_u16(self):
|
|
132
|
+
return self.read_bits(16)
|
|
133
|
+
|
|
134
|
+
def read_u32(self):
|
|
135
|
+
return self.read_bits(32)
|
|
136
|
+
|
|
137
|
+
def read_u64(self):
|
|
138
|
+
hi = self.read_u32()
|
|
139
|
+
lo = self.read_u32()
|
|
140
|
+
return hi * 0x100000000 + lo
|
|
141
|
+
|
|
142
|
+
def read_bytes(self, n):
|
|
143
|
+
out = bytearray(n)
|
|
144
|
+
for i in range(n):
|
|
145
|
+
out[i] = self.read_u8()
|
|
146
|
+
return bytes(out)
|
|
147
|
+
|
|
148
|
+
def peek_bits(self, n):
|
|
149
|
+
saved = self._pos
|
|
150
|
+
val = self.read_bits(n)
|
|
151
|
+
self._pos = saved
|
|
152
|
+
return val
|
|
153
|
+
|
|
154
|
+
def align_to_byte(self):
|
|
155
|
+
rem = self._pos & 7
|
|
156
|
+
if rem != 0:
|
|
157
|
+
self._pos += 8 - rem
|
|
158
|
+
|
|
159
|
+
def is_aligned(self):
|
|
160
|
+
return (self._pos & 7) == 0
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
"""CRC-32 computation — standard polynomial (same table as C integrity.c)."""
|
|
3
|
+
|
|
4
|
+
# fmt: off
|
|
5
|
+
TABLE = [
|
|
6
|
+
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
|
|
7
|
+
0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
|
|
8
|
+
0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
|
|
9
|
+
0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
|
|
10
|
+
0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
|
|
11
|
+
0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
|
|
12
|
+
0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
|
|
13
|
+
0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
|
|
14
|
+
0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
|
|
15
|
+
0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
|
|
16
|
+
0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
|
|
17
|
+
0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
|
|
18
|
+
0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
|
|
19
|
+
0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
|
|
20
|
+
0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
|
|
21
|
+
0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
|
|
22
|
+
0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
|
|
23
|
+
0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
|
|
24
|
+
0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
|
|
25
|
+
0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
|
|
26
|
+
0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
|
|
27
|
+
0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
|
|
28
|
+
0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
|
|
29
|
+
0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
|
|
30
|
+
0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
|
|
31
|
+
0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
|
|
32
|
+
0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
|
|
33
|
+
0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
|
|
34
|
+
0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
|
|
35
|
+
0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
|
|
36
|
+
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
|
|
37
|
+
0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
|
|
38
|
+
]
|
|
39
|
+
# fmt: on
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def crc32(data):
|
|
43
|
+
"""Compute CRC-32 over *data* (bytes-like)."""
|
|
44
|
+
crc = 0xFFFFFFFF
|
|
45
|
+
for b in data:
|
|
46
|
+
crc = TABLE[(crc ^ b) & 0xFF] ^ ((crc >> 8) & 0x00FFFFFF)
|
|
47
|
+
return (crc ^ 0xFFFFFFFF) & 0xFFFFFFFF
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
"""Trie decoder — port of src/core/decoder.c (via JavaScript decoder.js)."""
|
|
3
|
+
|
|
4
|
+
from .bitstream import BitReader
|
|
5
|
+
from .crc32 import crc32
|
|
6
|
+
from .values import decode_value
|
|
7
|
+
from .varint import read_var_uint
|
|
8
|
+
|
|
9
|
+
# Format constants
|
|
10
|
+
TP_MAGIC = bytes([0x54, 0x52, 0x50, 0x00])
|
|
11
|
+
TP_HEADER_SIZE = 32
|
|
12
|
+
TP_FLAG_HAS_VALUES = 1
|
|
13
|
+
NUM_CONTROL_CODES = 6
|
|
14
|
+
|
|
15
|
+
# Control code indices
|
|
16
|
+
CTRL_END = 0
|
|
17
|
+
CTRL_END_VAL = 1
|
|
18
|
+
CTRL_SKIP = 2
|
|
19
|
+
CTRL_BRANCH = 5
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def decode(buffer):
|
|
23
|
+
"""Decode a .trp binary buffer into a dict. Returns dict."""
|
|
24
|
+
if not isinstance(buffer, (bytes, bytearray)):
|
|
25
|
+
raise TypeError("decode expects bytes or bytearray")
|
|
26
|
+
if len(buffer) < TP_HEADER_SIZE + 4:
|
|
27
|
+
raise ValueError("Data too short for .trp format")
|
|
28
|
+
|
|
29
|
+
buf = bytes(buffer)
|
|
30
|
+
|
|
31
|
+
# Validate magic
|
|
32
|
+
for i in range(4):
|
|
33
|
+
if buf[i] != TP_MAGIC[i]:
|
|
34
|
+
raise ValueError("Invalid magic bytes — not a .trp file")
|
|
35
|
+
|
|
36
|
+
# Version
|
|
37
|
+
version_major = buf[4]
|
|
38
|
+
if version_major != 1:
|
|
39
|
+
raise ValueError(f"Unsupported format version: {version_major}")
|
|
40
|
+
|
|
41
|
+
# Flags
|
|
42
|
+
flags = (buf[6] << 8) | buf[7]
|
|
43
|
+
has_values = (flags & TP_FLAG_HAS_VALUES) != 0
|
|
44
|
+
|
|
45
|
+
# num_keys
|
|
46
|
+
num_keys = (buf[8] << 24) | (buf[9] << 16) | (buf[10] << 8) | buf[11]
|
|
47
|
+
|
|
48
|
+
# Offsets
|
|
49
|
+
trie_data_offset = (buf[12] << 24) | (buf[13] << 16) | (buf[14] << 8) | buf[15]
|
|
50
|
+
value_store_offset = (buf[16] << 24) | (buf[17] << 16) | (buf[18] << 8) | buf[19]
|
|
51
|
+
|
|
52
|
+
# CRC verification
|
|
53
|
+
crc_data_len = len(buf) - 4
|
|
54
|
+
expected_crc = (
|
|
55
|
+
(buf[crc_data_len] << 24)
|
|
56
|
+
| (buf[crc_data_len + 1] << 16)
|
|
57
|
+
| (buf[crc_data_len + 2] << 8)
|
|
58
|
+
| buf[crc_data_len + 3]
|
|
59
|
+
)
|
|
60
|
+
actual_crc = crc32(buf[:crc_data_len])
|
|
61
|
+
if actual_crc != expected_crc:
|
|
62
|
+
raise ValueError("CRC-32 integrity check failed")
|
|
63
|
+
|
|
64
|
+
if num_keys == 0:
|
|
65
|
+
return {}
|
|
66
|
+
|
|
67
|
+
# Parse trie config
|
|
68
|
+
reader = BitReader(buf)
|
|
69
|
+
data_start = TP_HEADER_SIZE * 8
|
|
70
|
+
|
|
71
|
+
reader.seek(data_start)
|
|
72
|
+
bps = reader.read_bits(4)
|
|
73
|
+
# Symbol codes index 256-entry maps, so a symbol may not be wider than a
|
|
74
|
+
# byte; 0 would make the trie unreadable.
|
|
75
|
+
if bps < 1 or bps > 8:
|
|
76
|
+
raise ValueError(f"Invalid trie config: bits_per_symbol must be 1..8, got {bps}")
|
|
77
|
+
symbol_count = reader.read_bits(8)
|
|
78
|
+
# Must leave room for the control codes and fit in bits_per_symbol.
|
|
79
|
+
if symbol_count < NUM_CONTROL_CODES or symbol_count > (1 << bps):
|
|
80
|
+
raise ValueError(f"Invalid trie config: symbol_count out of range: {symbol_count}")
|
|
81
|
+
|
|
82
|
+
# Read control codes
|
|
83
|
+
ctrl_codes = [0] * NUM_CONTROL_CODES
|
|
84
|
+
code_is_ctrl = [False] * 256
|
|
85
|
+
for c in range(NUM_CONTROL_CODES):
|
|
86
|
+
ctrl_codes[c] = reader.read_bits(bps)
|
|
87
|
+
if ctrl_codes[c] < 256:
|
|
88
|
+
code_is_ctrl[ctrl_codes[c]] = True
|
|
89
|
+
|
|
90
|
+
# Read symbol table
|
|
91
|
+
reverse_map = [0] * 256
|
|
92
|
+
for cd in range(NUM_CONTROL_CODES, symbol_count):
|
|
93
|
+
cp = read_var_uint(reader)
|
|
94
|
+
if cd < 256 and cp < 256:
|
|
95
|
+
reverse_map[cd] = cp
|
|
96
|
+
|
|
97
|
+
trie_start = data_start + trie_data_offset
|
|
98
|
+
value_start = data_start + value_store_offset
|
|
99
|
+
# The trie occupies [trie_start, value_start); the value store (when
|
|
100
|
+
# present), the byte padding and the CRC follow it.
|
|
101
|
+
trie_end = value_start
|
|
102
|
+
|
|
103
|
+
# DFS iteration
|
|
104
|
+
result = {}
|
|
105
|
+
key_stack = []
|
|
106
|
+
|
|
107
|
+
# Every subtree knows where it ends: a child with a SKIP ends at
|
|
108
|
+
# child_start + skip_dist, the last child ends where its parent does, and
|
|
109
|
+
# the root ends at trie_end. `end` is the only authority on whether more
|
|
110
|
+
# symbols belong to this subtree -- the bits that happen to follow are
|
|
111
|
+
# not, because past the last terminal they are padding and CRC.
|
|
112
|
+
def dfs_walk(r, end):
|
|
113
|
+
while True:
|
|
114
|
+
if r.position >= end:
|
|
115
|
+
return
|
|
116
|
+
sym = r.read_bits(bps)
|
|
117
|
+
|
|
118
|
+
if sym == ctrl_codes[CTRL_END]:
|
|
119
|
+
key_str = bytes(key_stack).decode("utf-8")
|
|
120
|
+
result[key_str] = None
|
|
121
|
+
walk_branch_if_present(r, end)
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
if sym == ctrl_codes[CTRL_END_VAL]:
|
|
125
|
+
read_var_uint(r) # value index
|
|
126
|
+
key_str = bytes(key_stack).decode("utf-8")
|
|
127
|
+
result[key_str] = None
|
|
128
|
+
walk_branch_if_present(r, end)
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
if sym == ctrl_codes[CTRL_BRANCH]:
|
|
132
|
+
child_count = read_var_uint(r)
|
|
133
|
+
walk_branch(r, child_count, end)
|
|
134
|
+
return
|
|
135
|
+
|
|
136
|
+
# Regular symbol
|
|
137
|
+
byte_val = reverse_map[sym] if sym < 256 else 0
|
|
138
|
+
key_stack.append(byte_val)
|
|
139
|
+
|
|
140
|
+
# A terminal is followed by a BRANCH exactly when the subtree has not
|
|
141
|
+
# reached its end -- keys that share this terminal as a prefix.
|
|
142
|
+
def walk_branch_if_present(r, end):
|
|
143
|
+
if r.position >= end:
|
|
144
|
+
return
|
|
145
|
+
sym = r.read_bits(bps)
|
|
146
|
+
if sym != ctrl_codes[CTRL_BRANCH]:
|
|
147
|
+
raise ValueError("Malformed trie: expected BRANCH after terminal")
|
|
148
|
+
child_count = read_var_uint(r)
|
|
149
|
+
walk_branch(r, child_count, end)
|
|
150
|
+
|
|
151
|
+
def walk_branch(r, child_count, end):
|
|
152
|
+
saved_key_len = len(key_stack)
|
|
153
|
+
for ci in range(child_count):
|
|
154
|
+
has_skip = ci < child_count - 1
|
|
155
|
+
skip_dist = 0
|
|
156
|
+
|
|
157
|
+
if has_skip:
|
|
158
|
+
r.read_bits(bps) # SKIP control code
|
|
159
|
+
skip_dist = read_var_uint(r)
|
|
160
|
+
|
|
161
|
+
child_start_pos = r.position
|
|
162
|
+
del key_stack[saved_key_len:]
|
|
163
|
+
dfs_walk(r, child_start_pos + skip_dist if has_skip else end)
|
|
164
|
+
|
|
165
|
+
if has_skip:
|
|
166
|
+
r.seek(child_start_pos + skip_dist)
|
|
167
|
+
|
|
168
|
+
del key_stack[saved_key_len:]
|
|
169
|
+
|
|
170
|
+
# Walk the trie
|
|
171
|
+
reader.seek(trie_start)
|
|
172
|
+
if num_keys > 0:
|
|
173
|
+
dfs_walk(reader, trie_end)
|
|
174
|
+
|
|
175
|
+
# Decode values
|
|
176
|
+
if has_values:
|
|
177
|
+
reader.seek(value_start)
|
|
178
|
+
sorted_keys = sorted(result.keys(), key=lambda k: k.encode("utf-8"))
|
|
179
|
+
for key in sorted_keys:
|
|
180
|
+
result[key] = decode_value(reader)
|
|
181
|
+
|
|
182
|
+
return result
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
"""Trie encoder — port of src/core/encoder.c (via JavaScript encoder.js)."""
|
|
3
|
+
|
|
4
|
+
from .bitstream import BitWriter
|
|
5
|
+
from .crc32 import crc32
|
|
6
|
+
from .values import encode_value
|
|
7
|
+
from .varint import var_uint_bits, write_var_uint
|
|
8
|
+
|
|
9
|
+
# Format constants (match core_internal.h)
|
|
10
|
+
TP_MAGIC = bytes([0x54, 0x52, 0x50, 0x00]) # "TRP\0"
|
|
11
|
+
TP_VERSION_MAJOR = 1
|
|
12
|
+
TP_VERSION_MINOR = 0
|
|
13
|
+
TP_HEADER_SIZE = 32
|
|
14
|
+
TP_FLAG_HAS_VALUES = 1
|
|
15
|
+
|
|
16
|
+
# Control code indices
|
|
17
|
+
CTRL_END = 0
|
|
18
|
+
CTRL_END_VAL = 1
|
|
19
|
+
CTRL_SKIP = 2
|
|
20
|
+
# CTRL_SUFFIX = 3
|
|
21
|
+
# CTRL_ESCAPE = 4
|
|
22
|
+
CTRL_BRANCH = 5
|
|
23
|
+
NUM_CONTROL_CODES = 6
|
|
24
|
+
|
|
25
|
+
# Largest number of distinct byte values the keys may use: symbol_count is an
|
|
26
|
+
# 8-bit header field holding the alphabet plus the control codes.
|
|
27
|
+
MAX_ALPHABET_SIZE = 255 - NUM_CONTROL_CODES # 249
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def encode(data):
|
|
31
|
+
"""Encode a dict into the .trp binary format. Returns bytes."""
|
|
32
|
+
if not isinstance(data, dict):
|
|
33
|
+
raise TypeError("encode expects a dict")
|
|
34
|
+
|
|
35
|
+
# Collect entries: (key_bytes, val)
|
|
36
|
+
entries = []
|
|
37
|
+
for k, v in data.items():
|
|
38
|
+
key_bytes = k.encode("utf-8") if isinstance(k, str) else bytes(k)
|
|
39
|
+
entries.append((key_bytes, v))
|
|
40
|
+
|
|
41
|
+
# Sort lexicographically by key bytes
|
|
42
|
+
entries.sort(key=lambda e: e[0])
|
|
43
|
+
|
|
44
|
+
deduped = entries # dict keys are unique; no dedup needed
|
|
45
|
+
|
|
46
|
+
# Determine if any entries have non-null values
|
|
47
|
+
has_values = any(e[1] is not None for e in deduped)
|
|
48
|
+
|
|
49
|
+
# Symbol analysis
|
|
50
|
+
used = [False] * 256
|
|
51
|
+
for kb, _ in deduped:
|
|
52
|
+
for b in kb:
|
|
53
|
+
used[b] = True
|
|
54
|
+
|
|
55
|
+
alphabet_size = sum(used)
|
|
56
|
+
total_symbols = alphabet_size + NUM_CONTROL_CODES
|
|
57
|
+
# The trie config packs symbol_count into 8 header bits, so the alphabet
|
|
58
|
+
# plus the 6 control codes must fit in 255. Encoding a wider alphabet used
|
|
59
|
+
# to produce a buffer with a valid CRC that decoded to nothing.
|
|
60
|
+
if total_symbols > 255:
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"Keys use too many distinct byte values ({alphabet_size}); "
|
|
63
|
+
f"the format allows at most {MAX_ALPHABET_SIZE}"
|
|
64
|
+
)
|
|
65
|
+
bps = 1
|
|
66
|
+
while (1 << bps) < total_symbols:
|
|
67
|
+
bps += 1
|
|
68
|
+
|
|
69
|
+
# Control codes: 0..5
|
|
70
|
+
ctrl_codes = list(range(NUM_CONTROL_CODES))
|
|
71
|
+
|
|
72
|
+
# Symbol map: byte value -> N-bit code
|
|
73
|
+
symbol_map = [0] * 256
|
|
74
|
+
reverse_map = [0] * 256
|
|
75
|
+
code = NUM_CONTROL_CODES
|
|
76
|
+
for i in range(256):
|
|
77
|
+
if used[i]:
|
|
78
|
+
symbol_map[i] = code
|
|
79
|
+
if code < 256:
|
|
80
|
+
reverse_map[code] = i
|
|
81
|
+
code += 1
|
|
82
|
+
|
|
83
|
+
# Build the bitstream
|
|
84
|
+
w = BitWriter(256)
|
|
85
|
+
|
|
86
|
+
# Write 32-byte header placeholder
|
|
87
|
+
for b in TP_MAGIC:
|
|
88
|
+
w.write_u8(b)
|
|
89
|
+
w.write_u8(TP_VERSION_MAJOR)
|
|
90
|
+
w.write_u8(TP_VERSION_MINOR)
|
|
91
|
+
flags = TP_FLAG_HAS_VALUES if has_values else 0
|
|
92
|
+
w.write_u16(flags)
|
|
93
|
+
w.write_u32(len(deduped)) # num_keys
|
|
94
|
+
w.write_u32(0) # trie_data_offset placeholder
|
|
95
|
+
w.write_u32(0) # value_store_offset placeholder
|
|
96
|
+
w.write_u32(0) # suffix_table_offset placeholder
|
|
97
|
+
w.write_u32(0) # total_data_bits placeholder
|
|
98
|
+
w.write_u32(0) # reserved
|
|
99
|
+
|
|
100
|
+
data_start = w.position # should be 256 (32 bytes * 8)
|
|
101
|
+
|
|
102
|
+
# Trie config: bits_per_symbol (4 bits) + symbol_count (8 bits)
|
|
103
|
+
w.write_bits(bps, 4)
|
|
104
|
+
w.write_bits(total_symbols, 8)
|
|
105
|
+
|
|
106
|
+
# Control code mappings
|
|
107
|
+
for c in range(NUM_CONTROL_CODES):
|
|
108
|
+
w.write_bits(ctrl_codes[c], bps)
|
|
109
|
+
|
|
110
|
+
# Symbol table
|
|
111
|
+
for cd in range(NUM_CONTROL_CODES, total_symbols):
|
|
112
|
+
byte_val = reverse_map[cd] if cd < 256 else 0
|
|
113
|
+
write_var_uint(w, byte_val)
|
|
114
|
+
|
|
115
|
+
trie_data_offset = w.position - data_start
|
|
116
|
+
|
|
117
|
+
# Write prefix trie
|
|
118
|
+
ctx = {
|
|
119
|
+
"deduped": deduped,
|
|
120
|
+
"bps": bps,
|
|
121
|
+
"symbol_map": symbol_map,
|
|
122
|
+
"ctrl_codes": ctrl_codes,
|
|
123
|
+
"has_values": has_values,
|
|
124
|
+
}
|
|
125
|
+
value_idx = [0]
|
|
126
|
+
if deduped:
|
|
127
|
+
_trie_write(ctx, w, 0, len(deduped), 0, value_idx)
|
|
128
|
+
|
|
129
|
+
value_store_offset = w.position - data_start
|
|
130
|
+
|
|
131
|
+
# Write value store
|
|
132
|
+
if has_values:
|
|
133
|
+
for _, val in deduped:
|
|
134
|
+
encode_value(w, val)
|
|
135
|
+
|
|
136
|
+
total_data_bits = w.position - data_start
|
|
137
|
+
|
|
138
|
+
w.align_to_byte()
|
|
139
|
+
|
|
140
|
+
# Get pre-CRC buffer
|
|
141
|
+
pre_crc_buf = bytearray(w.to_bytes())
|
|
142
|
+
crc_val = crc32(pre_crc_buf)
|
|
143
|
+
|
|
144
|
+
# Append CRC
|
|
145
|
+
w.write_u32(crc_val)
|
|
146
|
+
|
|
147
|
+
out_buf = bytearray(w.to_bytes())
|
|
148
|
+
|
|
149
|
+
# Patch header offsets
|
|
150
|
+
_patch_u32(out_buf, 12, trie_data_offset)
|
|
151
|
+
_patch_u32(out_buf, 16, value_store_offset)
|
|
152
|
+
_patch_u32(out_buf, 20, 0) # suffix_table_offset
|
|
153
|
+
_patch_u32(out_buf, 24, total_data_bits)
|
|
154
|
+
|
|
155
|
+
# Recompute CRC over patched data
|
|
156
|
+
crc_data_len = len(out_buf) - 4
|
|
157
|
+
crc_val = crc32(out_buf[:crc_data_len])
|
|
158
|
+
_patch_u32(out_buf, crc_data_len, crc_val)
|
|
159
|
+
|
|
160
|
+
return bytes(out_buf)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _patch_u32(buf, offset, value):
|
|
164
|
+
buf[offset] = (value >> 24) & 0xFF
|
|
165
|
+
buf[offset + 1] = (value >> 16) & 0xFF
|
|
166
|
+
buf[offset + 2] = (value >> 8) & 0xFF
|
|
167
|
+
buf[offset + 3] = value & 0xFF
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _trie_subtree_size(ctx, start, end, prefix_len, value_idx):
|
|
171
|
+
"""Compute subtree size in bits (dry run)."""
|
|
172
|
+
deduped = ctx["deduped"]
|
|
173
|
+
bps = ctx["bps"]
|
|
174
|
+
has_values = ctx["has_values"]
|
|
175
|
+
bits = 0
|
|
176
|
+
|
|
177
|
+
# Find common prefix beyond prefix_len
|
|
178
|
+
common = prefix_len
|
|
179
|
+
while True:
|
|
180
|
+
all_have = True
|
|
181
|
+
ch = 0
|
|
182
|
+
for i in range(start, end):
|
|
183
|
+
if len(deduped[i][0]) <= common:
|
|
184
|
+
all_have = False
|
|
185
|
+
break
|
|
186
|
+
if i == start:
|
|
187
|
+
ch = deduped[i][0][common]
|
|
188
|
+
elif deduped[i][0][common] != ch:
|
|
189
|
+
all_have = False
|
|
190
|
+
break
|
|
191
|
+
if not all_have:
|
|
192
|
+
break
|
|
193
|
+
common += 1
|
|
194
|
+
|
|
195
|
+
bits += (common - prefix_len) * bps
|
|
196
|
+
|
|
197
|
+
has_terminal = len(deduped[start][0]) == common
|
|
198
|
+
children_start = start + 1 if has_terminal else start
|
|
199
|
+
|
|
200
|
+
# Count children
|
|
201
|
+
child_count = 0
|
|
202
|
+
cs = children_start
|
|
203
|
+
while cs < end:
|
|
204
|
+
child_count += 1
|
|
205
|
+
ch = deduped[cs][0][common]
|
|
206
|
+
ce = cs + 1
|
|
207
|
+
while ce < end and deduped[ce][0][common] == ch:
|
|
208
|
+
ce += 1
|
|
209
|
+
cs = ce
|
|
210
|
+
|
|
211
|
+
if has_terminal and child_count == 0:
|
|
212
|
+
if has_values and deduped[start][1] is not None:
|
|
213
|
+
bits += bps
|
|
214
|
+
bits += var_uint_bits(value_idx[0])
|
|
215
|
+
else:
|
|
216
|
+
bits += bps
|
|
217
|
+
value_idx[0] += 1
|
|
218
|
+
elif has_terminal and child_count > 0:
|
|
219
|
+
if has_values and deduped[start][1] is not None:
|
|
220
|
+
bits += bps
|
|
221
|
+
bits += var_uint_bits(value_idx[0])
|
|
222
|
+
else:
|
|
223
|
+
bits += bps
|
|
224
|
+
value_idx[0] += 1
|
|
225
|
+
bits += bps
|
|
226
|
+
bits += var_uint_bits(child_count)
|
|
227
|
+
|
|
228
|
+
cs = children_start
|
|
229
|
+
child_i = 0
|
|
230
|
+
while cs < end:
|
|
231
|
+
ch = deduped[cs][0][common]
|
|
232
|
+
ce = cs + 1
|
|
233
|
+
while ce < end and deduped[ce][0][common] == ch:
|
|
234
|
+
ce += 1
|
|
235
|
+
if child_i < child_count - 1:
|
|
236
|
+
child_sz = _trie_subtree_size(ctx, cs, ce, common, value_idx)
|
|
237
|
+
bits += bps
|
|
238
|
+
bits += var_uint_bits(child_sz)
|
|
239
|
+
bits += child_sz
|
|
240
|
+
else:
|
|
241
|
+
bits += _trie_subtree_size(ctx, cs, ce, common, value_idx)
|
|
242
|
+
child_i += 1
|
|
243
|
+
cs = ce
|
|
244
|
+
else: # not has_terminal, child_count > 1
|
|
245
|
+
bits += bps
|
|
246
|
+
bits += var_uint_bits(child_count)
|
|
247
|
+
|
|
248
|
+
cs = children_start
|
|
249
|
+
child_i = 0
|
|
250
|
+
while cs < end:
|
|
251
|
+
ch = deduped[cs][0][common]
|
|
252
|
+
ce = cs + 1
|
|
253
|
+
while ce < end and deduped[ce][0][common] == ch:
|
|
254
|
+
ce += 1
|
|
255
|
+
if child_i < child_count - 1:
|
|
256
|
+
child_sz = _trie_subtree_size(ctx, cs, ce, common, value_idx)
|
|
257
|
+
bits += bps
|
|
258
|
+
bits += var_uint_bits(child_sz)
|
|
259
|
+
bits += child_sz
|
|
260
|
+
else:
|
|
261
|
+
bits += _trie_subtree_size(ctx, cs, ce, common, value_idx)
|
|
262
|
+
child_i += 1
|
|
263
|
+
cs = ce
|
|
264
|
+
|
|
265
|
+
return bits
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _trie_write(ctx, w, start, end, prefix_len, value_idx):
|
|
269
|
+
"""Write the trie subtree."""
|
|
270
|
+
deduped = ctx["deduped"]
|
|
271
|
+
bps = ctx["bps"]
|
|
272
|
+
symbol_map = ctx["symbol_map"]
|
|
273
|
+
ctrl_codes = ctx["ctrl_codes"]
|
|
274
|
+
has_values = ctx["has_values"]
|
|
275
|
+
|
|
276
|
+
# Find common prefix beyond prefix_len
|
|
277
|
+
common = prefix_len
|
|
278
|
+
while True:
|
|
279
|
+
all_have = True
|
|
280
|
+
ch = 0
|
|
281
|
+
for i in range(start, end):
|
|
282
|
+
if len(deduped[i][0]) <= common:
|
|
283
|
+
all_have = False
|
|
284
|
+
break
|
|
285
|
+
if i == start:
|
|
286
|
+
ch = deduped[i][0][common]
|
|
287
|
+
elif deduped[i][0][common] != ch:
|
|
288
|
+
all_have = False
|
|
289
|
+
break
|
|
290
|
+
if not all_have:
|
|
291
|
+
break
|
|
292
|
+
common += 1
|
|
293
|
+
|
|
294
|
+
# Write common prefix symbols
|
|
295
|
+
for i in range(prefix_len, common):
|
|
296
|
+
ch = deduped[start][0][i]
|
|
297
|
+
w.write_bits(symbol_map[ch], bps)
|
|
298
|
+
|
|
299
|
+
has_terminal = len(deduped[start][0]) == common
|
|
300
|
+
children_start = start + 1 if has_terminal else start
|
|
301
|
+
|
|
302
|
+
# Count children
|
|
303
|
+
child_count = 0
|
|
304
|
+
cs = children_start
|
|
305
|
+
while cs < end:
|
|
306
|
+
child_count += 1
|
|
307
|
+
ch = deduped[cs][0][common]
|
|
308
|
+
ce = cs + 1
|
|
309
|
+
while ce < end and deduped[ce][0][common] == ch:
|
|
310
|
+
ce += 1
|
|
311
|
+
cs = ce
|
|
312
|
+
|
|
313
|
+
# Write terminal
|
|
314
|
+
if has_terminal:
|
|
315
|
+
if has_values and deduped[start][1] is not None:
|
|
316
|
+
w.write_bits(ctrl_codes[CTRL_END_VAL], bps)
|
|
317
|
+
write_var_uint(w, value_idx[0])
|
|
318
|
+
else:
|
|
319
|
+
w.write_bits(ctrl_codes[CTRL_END], bps)
|
|
320
|
+
value_idx[0] += 1
|
|
321
|
+
|
|
322
|
+
if child_count == 0:
|
|
323
|
+
return
|
|
324
|
+
|
|
325
|
+
# BRANCH
|
|
326
|
+
w.write_bits(ctrl_codes[CTRL_BRANCH], bps)
|
|
327
|
+
write_var_uint(w, child_count)
|
|
328
|
+
|
|
329
|
+
cs = children_start
|
|
330
|
+
child_i = 0
|
|
331
|
+
while cs < end:
|
|
332
|
+
ch = deduped[cs][0][common]
|
|
333
|
+
ce = cs + 1
|
|
334
|
+
while ce < end and deduped[ce][0][common] == ch:
|
|
335
|
+
ce += 1
|
|
336
|
+
|
|
337
|
+
if child_i < child_count - 1:
|
|
338
|
+
vi_copy = [value_idx[0]]
|
|
339
|
+
child_sz = _trie_subtree_size(ctx, cs, ce, common, vi_copy)
|
|
340
|
+
w.write_bits(ctrl_codes[CTRL_SKIP], bps)
|
|
341
|
+
write_var_uint(w, child_sz)
|
|
342
|
+
|
|
343
|
+
_trie_write(ctx, w, cs, ce, common, value_idx)
|
|
344
|
+
|
|
345
|
+
child_i += 1
|
|
346
|
+
cs = ce
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
"""Value encoder/decoder for 8 type tags — matches src/core/value.c."""
|
|
3
|
+
|
|
4
|
+
import struct
|
|
5
|
+
|
|
6
|
+
from .varint import read_var_int, read_var_uint, write_var_int, write_var_uint
|
|
7
|
+
|
|
8
|
+
# Type tags (4-bit, matches tp_value_type enum)
|
|
9
|
+
TP_NULL = 0
|
|
10
|
+
TP_BOOL = 1
|
|
11
|
+
TP_INT = 2
|
|
12
|
+
TP_UINT = 3
|
|
13
|
+
TP_FLOAT32 = 4
|
|
14
|
+
TP_FLOAT64 = 5
|
|
15
|
+
TP_STRING = 6
|
|
16
|
+
TP_BLOB = 7
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def encode_value(writer, val):
|
|
20
|
+
"""Encode a Python value into the bitstream."""
|
|
21
|
+
if val is None:
|
|
22
|
+
writer.write_bits(TP_NULL, 4)
|
|
23
|
+
return
|
|
24
|
+
|
|
25
|
+
if isinstance(val, bool):
|
|
26
|
+
writer.write_bits(TP_BOOL, 4)
|
|
27
|
+
writer.write_bit(1 if val else 0)
|
|
28
|
+
return
|
|
29
|
+
|
|
30
|
+
if isinstance(val, int):
|
|
31
|
+
if val < 0:
|
|
32
|
+
writer.write_bits(TP_INT, 4)
|
|
33
|
+
write_var_int(writer, val)
|
|
34
|
+
else:
|
|
35
|
+
writer.write_bits(TP_UINT, 4)
|
|
36
|
+
write_var_uint(writer, val)
|
|
37
|
+
return
|
|
38
|
+
|
|
39
|
+
if isinstance(val, float):
|
|
40
|
+
writer.write_bits(TP_FLOAT64, 4)
|
|
41
|
+
packed = struct.pack(">d", val)
|
|
42
|
+
hi = struct.unpack(">I", packed[:4])[0]
|
|
43
|
+
lo = struct.unpack(">I", packed[4:])[0]
|
|
44
|
+
writer.write_u64(hi, lo)
|
|
45
|
+
return
|
|
46
|
+
|
|
47
|
+
if isinstance(val, str):
|
|
48
|
+
encoded = val.encode("utf-8")
|
|
49
|
+
writer.write_bits(TP_STRING, 4)
|
|
50
|
+
write_var_uint(writer, len(encoded))
|
|
51
|
+
writer.align_to_byte()
|
|
52
|
+
writer.write_bytes(encoded)
|
|
53
|
+
return
|
|
54
|
+
|
|
55
|
+
if isinstance(val, (bytes, bytearray)):
|
|
56
|
+
writer.write_bits(TP_BLOB, 4)
|
|
57
|
+
write_var_uint(writer, len(val))
|
|
58
|
+
writer.align_to_byte()
|
|
59
|
+
writer.write_bytes(val)
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
raise TypeError(f"Unsupported value type: {type(val)}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def decode_value(reader):
|
|
66
|
+
"""Decode a value from the bitstream."""
|
|
67
|
+
tag = reader.read_bits(4)
|
|
68
|
+
|
|
69
|
+
if tag == TP_NULL:
|
|
70
|
+
return None
|
|
71
|
+
if tag == TP_BOOL:
|
|
72
|
+
return reader.read_bit() != 0
|
|
73
|
+
if tag == TP_INT:
|
|
74
|
+
return read_var_int(reader)
|
|
75
|
+
if tag == TP_UINT:
|
|
76
|
+
return read_var_uint(reader)
|
|
77
|
+
if tag == TP_FLOAT32:
|
|
78
|
+
bits = reader.read_u32()
|
|
79
|
+
packed = struct.pack(">I", bits)
|
|
80
|
+
return struct.unpack(">f", packed)[0]
|
|
81
|
+
if tag == TP_FLOAT64:
|
|
82
|
+
hi = reader.read_u32()
|
|
83
|
+
lo = reader.read_u32()
|
|
84
|
+
packed = struct.pack(">II", hi, lo)
|
|
85
|
+
return struct.unpack(">d", packed)[0]
|
|
86
|
+
if tag == TP_STRING:
|
|
87
|
+
slen = read_var_uint(reader)
|
|
88
|
+
reader.align_to_byte()
|
|
89
|
+
raw = reader.read_bytes(slen)
|
|
90
|
+
return raw.decode("utf-8")
|
|
91
|
+
if tag == TP_BLOB:
|
|
92
|
+
blen = read_var_uint(reader)
|
|
93
|
+
reader.align_to_byte()
|
|
94
|
+
return reader.read_bytes(blen)
|
|
95
|
+
|
|
96
|
+
raise ValueError(f"Unknown value type tag: {tag}")
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Copyright (c) 2026 M. A. Chatterjee, BSD-2-Clause.
|
|
2
|
+
"""LEB128 unsigned VarInt + zigzag signed VarInt — matches bitstream_varint.c."""
|
|
3
|
+
|
|
4
|
+
VARINT_MAX_GROUPS = 10
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def write_var_uint(writer, value):
|
|
8
|
+
"""Write an unsigned LEB128 VarInt."""
|
|
9
|
+
if value < 0:
|
|
10
|
+
raise ValueError("write_var_uint: negative value")
|
|
11
|
+
while True:
|
|
12
|
+
byte = value & 0x7F
|
|
13
|
+
value >>= 7
|
|
14
|
+
if value > 0:
|
|
15
|
+
byte |= 0x80
|
|
16
|
+
writer.write_u8(byte)
|
|
17
|
+
if value == 0:
|
|
18
|
+
break
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def read_var_uint(reader):
|
|
22
|
+
"""Read an unsigned LEB128 VarInt."""
|
|
23
|
+
val = 0
|
|
24
|
+
shift = 0
|
|
25
|
+
for _ in range(VARINT_MAX_GROUPS):
|
|
26
|
+
byte = reader.read_u8()
|
|
27
|
+
val |= (byte & 0x7F) << shift
|
|
28
|
+
if (byte & 0x80) == 0:
|
|
29
|
+
return val
|
|
30
|
+
shift += 7
|
|
31
|
+
raise OverflowError("VarInt overflow")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def write_var_int(writer, value):
|
|
35
|
+
"""Write a signed zigzag VarInt."""
|
|
36
|
+
if value >= 0:
|
|
37
|
+
raw = value * 2
|
|
38
|
+
else:
|
|
39
|
+
raw = (-value) * 2 - 1
|
|
40
|
+
write_var_uint(writer, raw)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def read_var_int(reader):
|
|
44
|
+
"""Read a signed zigzag VarInt."""
|
|
45
|
+
raw = read_var_uint(reader)
|
|
46
|
+
if raw & 1:
|
|
47
|
+
return -(raw >> 1) - 1
|
|
48
|
+
return raw >> 1
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def var_uint_bits(val):
|
|
52
|
+
"""Return the number of bits needed to encode val as a VarInt."""
|
|
53
|
+
bits = 0
|
|
54
|
+
while True:
|
|
55
|
+
bits += 8
|
|
56
|
+
val >>= 7
|
|
57
|
+
if val == 0:
|
|
58
|
+
break
|
|
59
|
+
return bits
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: triepack
|
|
3
|
+
Version: 1.3.1
|
|
4
|
+
Summary: Compressed trie dictionary format (.trp) — compact binary key-value storage with fast lookups and prefix search. Pure Python, no dependencies.
|
|
5
|
+
Author-email: "M. A. Chatterjee" <deftio@deftio.com>
|
|
6
|
+
License-Expression: BSD-2-Clause
|
|
7
|
+
Project-URL: Homepage, https://deftio.github.io/triepack/
|
|
8
|
+
Project-URL: Documentation, https://deftio.github.io/triepack/guide/api-reference/
|
|
9
|
+
Project-URL: Repository, https://github.com/deftio/triepack
|
|
10
|
+
Project-URL: Issues, https://github.com/deftio/triepack/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/deftio/triepack/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: trie,prefix-tree,dictionary,binary-format,serialization,compression,key-value,prefix-search,trp,triepack
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
24
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Classifier: Topic :: System :: Archiving :: Compression
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.8
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENSE.txt
|
|
31
|
+
Provides-Extra: test
|
|
32
|
+
Requires-Dist: pytest; extra == "test"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# triepack
|
|
36
|
+
|
|
37
|
+
Compressed trie dictionary format (`.trp`) — compact binary key-value storage
|
|
38
|
+
with fast lookups and prefix search.
|
|
39
|
+
|
|
40
|
+
This is the native Python implementation: pure Python, no dependencies, no C
|
|
41
|
+
extension to build. It reads and writes the same bytes as the C reference
|
|
42
|
+
library and the JavaScript, Go, Rust, Swift, Java and Kotlin implementations.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install triepack
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Use
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from triepack import encode, decode
|
|
54
|
+
|
|
55
|
+
buf = encode({"hello": 42, "world": "foo"})
|
|
56
|
+
# buf is bytes holding the .trp binary
|
|
57
|
+
|
|
58
|
+
result = decode(buf)
|
|
59
|
+
print(result) # {'hello': 42, 'world': 'foo'}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Values
|
|
63
|
+
|
|
64
|
+
| Python | `.trp` type |
|
|
65
|
+
|-------------------|--------------------|
|
|
66
|
+
| `None` | null |
|
|
67
|
+
| `bool` | bool |
|
|
68
|
+
| `int >= 0` | uint |
|
|
69
|
+
| `int < 0` | int |
|
|
70
|
+
| `float` | float64 |
|
|
71
|
+
| `str` | string (UTF-8) |
|
|
72
|
+
| `bytes` | blob |
|
|
73
|
+
|
|
74
|
+
Decoding also accepts float32 values written by other implementations,
|
|
75
|
+
widening them to a Python float. Python integers are arbitrary precision, so
|
|
76
|
+
the full 64-bit signed and unsigned ranges round-trip exactly.
|
|
77
|
+
|
|
78
|
+
## Format
|
|
79
|
+
|
|
80
|
+
Every buffer carries a 32-byte header, a bit-packed prefix trie, a typed value
|
|
81
|
+
store and a CRC-32. Keys are stored once per shared prefix, and symbols are
|
|
82
|
+
packed at the minimum width the key alphabet needs.
|
|
83
|
+
|
|
84
|
+
The encoder is deterministic: the same input produces the same bytes in every
|
|
85
|
+
implementation, which is checked by a
|
|
86
|
+
[shared conformance suite](https://github.com/deftio/triepack/tree/main/tests/conformance)
|
|
87
|
+
that all nine implementations run.
|
|
88
|
+
|
|
89
|
+
## Links
|
|
90
|
+
|
|
91
|
+
- [Documentation](https://deftio.github.io/triepack/)
|
|
92
|
+
- [API reference](https://deftio.github.io/triepack/guide/api-reference/)
|
|
93
|
+
- [Source and issues](https://github.com/deftio/triepack)
|
|
94
|
+
- [Changelog](https://github.com/deftio/triepack/blob/main/CHANGELOG.md)
|
|
95
|
+
|
|
96
|
+
## Development
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
cd bindings/python
|
|
100
|
+
pip install -e ".[test]"
|
|
101
|
+
python -m pytest
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## License
|
|
105
|
+
|
|
106
|
+
Copyright (c) 2026 M. A. Chatterjee. BSD-2-Clause — see
|
|
107
|
+
[LICENSE.txt](LICENSE.txt).
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE.txt
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
setup.py
|
|
6
|
+
triepack/__init__.py
|
|
7
|
+
triepack/bitstream.py
|
|
8
|
+
triepack/crc32.py
|
|
9
|
+
triepack/decoder.py
|
|
10
|
+
triepack/encoder.py
|
|
11
|
+
triepack/values.py
|
|
12
|
+
triepack/varint.py
|
|
13
|
+
triepack.egg-info/PKG-INFO
|
|
14
|
+
triepack.egg-info/SOURCES.txt
|
|
15
|
+
triepack.egg-info/dependency_links.txt
|
|
16
|
+
triepack.egg-info/requires.txt
|
|
17
|
+
triepack.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
triepack
|