hashcodecs 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/Cargo.lock +1 -1
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/Cargo.toml +2 -2
- hashcodecs-0.2.0/PKG-INFO +179 -0
- hashcodecs-0.2.0/README.md +165 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/benches/base64.rs +6 -10
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/benches/murmur3.rs +5 -5
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/build.rs +2 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/pyproject.toml +7 -5
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/python/hashcodecs/__init__.py +11 -1
- hashcodecs-0.2.0/python/hashcodecs/_hashcodecs.pyi +50 -0
- hashcodecs-0.2.0/python/hashcodecs/murmur3.py +19 -0
- hashcodecs-0.2.0/src/base64/aarch64.rs +237 -0
- hashcodecs-0.2.0/src/base64/dispatch.rs +310 -0
- hashcodecs-0.2.0/src/base64/x86.rs +643 -0
- hashcodecs-0.2.0/src/base64/x86_avx512.rs +184 -0
- hashcodecs-0.2.0/src/base64.rs +555 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/src/lib.rs +5 -2
- hashcodecs-0.2.0/src/murmur3/dispatch.rs +47 -0
- hashcodecs-0.2.0/src/murmur3/x86.rs +472 -0
- hashcodecs-0.2.0/src/murmur3.rs +883 -0
- hashcodecs-0.1.0/src/python.rs → hashcodecs-0.2.0/src/python/base64.rs +13 -160
- hashcodecs-0.2.0/src/python/buffer.rs +88 -0
- hashcodecs-0.2.0/src/python/murmur3.rs +275 -0
- hashcodecs-0.2.0/src/python.rs +27 -0
- hashcodecs-0.1.0/PKG-INFO +0 -141
- hashcodecs-0.1.0/README.md +0 -127
- hashcodecs-0.1.0/benchmarks/python_base64.py +0 -200
- hashcodecs-0.1.0/python/hashcodecs/_hashcodecs.pyi +0 -11
- hashcodecs-0.1.0/python/hashcodecs/murmur3.py +0 -9
- hashcodecs-0.1.0/src/base64.rs +0 -1603
- hashcodecs-0.1.0/src/murmur3.rs +0 -337
- hashcodecs-0.1.0/tests/test_python_api.py +0 -156
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/.gitignore +0 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/LICENSE +0 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/LICENSE-MIT +0 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/benches/support/mod.rs +0 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/hatch_build.py +0 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/python/hashcodecs/base64.py +0 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/python/hashcodecs/base64.pyi +0 -0
- {hashcodecs-0.1.0 → hashcodecs-0.2.0}/python/hashcodecs/py.typed +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "hashcodecs"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
edition = "2024"
|
|
5
|
-
rust-version = "1.
|
|
5
|
+
rust-version = "1.89"
|
|
6
6
|
description = "SIMD-accelerated Base64 codecs and fast MurmurHash3 implementations"
|
|
7
7
|
license = "MIT OR Apache-2.0"
|
|
8
8
|
repository = "https://github.com/kozistr/hashcodecs-rs"
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hashcodecs
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: SIMD-accelerated Base64 and MurmurHash3 codecs
|
|
5
|
+
License-Expression: MIT OR Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
License-File: LICENSE-MIT
|
|
8
|
+
Classifier: Operating System :: OS Independent
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Rust
|
|
11
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# hashcodecs
|
|
16
|
+
|
|
17
|
+
[](https://github.com/kozistr/hashcodecs-rs/actions/workflows/ci.yml)
|
|
18
|
+
[](https://pypi.org/project/hashcodecs/)
|
|
19
|
+
[](https://pypi.org/project/hashcodecs/)
|
|
20
|
+
[](https://github.com/kozistr/hashcodecs-rs#license)
|
|
21
|
+
[](https://pypi.org/project/hashcodecs/)
|
|
22
|
+
|
|
23
|
+
`hashcodecs` provides runtime-dispatched SIMD Base64 codecs and fast, reference-compatible MurmurHash3 functions for Rust and Python.
|
|
24
|
+
|
|
25
|
+
## Design
|
|
26
|
+
|
|
27
|
+
- Base64 is implemented in this crate with runtime AVX-512 VBMI, AVX2, SSE4.1, SSSE3, and AArch64 NEON dispatch. Portable wheels select the best available SIMD backend and fall back to scalar code otherwise.
|
|
28
|
+
- Rust callers can reuse their own output buffers through the `*_into` APIs. Every backend writes exactly the documented prefix; it never depends on spare capacity after the destination slice.
|
|
29
|
+
- Rust-owned results use the `mimalloc` global allocator.
|
|
30
|
+
- Every MurmurHash3 variant uses runtime-dispatched AVX2 or SSE4.1 pre-mixing where its measured crossover beats the scalar loop. The x64 128-bit AVX2 kernel also selects BMI2 rotations when available. Ordered state transitions remain reference-compatible, and every variant has a portable scalar fallback.
|
|
31
|
+
- The Python Base64 surface follows the familiar `base64` names. Use `import hashcodecs.base64 as base64` when replacing standard Base64 calls.
|
|
32
|
+
- Large Python calls release the GIL. Strict decoding and valid default-mode decoding write directly into the returned `bytes`; malformed default-mode input uses a CPython-compatible lenient fallback.
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
pip3 install hashcodecs
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Usage
|
|
41
|
+
|
|
42
|
+
### Rust
|
|
43
|
+
|
|
44
|
+
```rust
|
|
45
|
+
let encoded = hashcodecs::b64encode(b"hello");
|
|
46
|
+
assert_eq!(encoded, "aGVsbG8=");
|
|
47
|
+
|
|
48
|
+
let mut output = [0_u8; 8];
|
|
49
|
+
let written = hashcodecs::b64encode_into(b"hello", &mut output).unwrap();
|
|
50
|
+
assert_eq!(&output[..written], b"aGVsbG8=");
|
|
51
|
+
assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### Python
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
import hashcodecs.base64 as base64
|
|
58
|
+
from hashcodecs import murmur3_32, murmur3_x64_128
|
|
59
|
+
|
|
60
|
+
assert base64.b64encode(b"hello") == b"aGVsbG8="
|
|
61
|
+
assert base64.b64decode(b"aGVsbG8=") == b"hello"
|
|
62
|
+
assert murmur3_32(b"hello") == 0x248BFA47
|
|
63
|
+
|
|
64
|
+
hasher = murmur3_x64_128(seed=42)
|
|
65
|
+
hasher.update(b"hello")
|
|
66
|
+
assert hasher.hexdigest() == hasher.digest().hex()
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Build an installable wheel and source distribution with:
|
|
70
|
+
|
|
71
|
+
```sh
|
|
72
|
+
uv build
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
# Benchmark
|
|
76
|
+
|
|
77
|
+
Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
|
|
78
|
+
|
|
79
|
+
Conditions: clean builds, one pinned logical CPU, single-threaded execution, and output allocation included. Higher is better.
|
|
80
|
+
|
|
81
|
+
## Run Locally
|
|
82
|
+
|
|
83
|
+
Comparison crates are development-only dependencies and are not included in consumer builds.
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
cargo bench --bench base64
|
|
87
|
+
cargo bench --bench murmur3
|
|
88
|
+
uv sync --group benchmark --no-install-project
|
|
89
|
+
uv run --no-project python benchmarks/python_base64.py
|
|
90
|
+
uv run --no-project python benchmarks/python_murmur3.py
|
|
91
|
+
uv run --no-project python benchmarks/python_murmur3.py --incremental
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
To time only this implementation:
|
|
95
|
+
|
|
96
|
+
```sh
|
|
97
|
+
cargo bench --bench base64 -- hashcodecs
|
|
98
|
+
cargo bench --bench murmur3 -- hashcodecs
|
|
99
|
+
uv run --no-project python benchmarks/python_base64.py --hashcodecs-only
|
|
100
|
+
uv run --no-project python benchmarks/python_murmur3.py --hashcodecs-only
|
|
101
|
+
uv run --no-project python benchmarks/python_murmur3.py --incremental --hashcodecs-only
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Base64: Rust
|
|
105
|
+
|
|
106
|
+
| Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
|
|
107
|
+
| --- | --- | --- | ---: | ---: | ---: |
|
|
108
|
+
| Standard | 4 KiB | encode | **18.17 GiB/s** | 5.40 GiB/s | 16.97 GiB/s |
|
|
109
|
+
| | 4 KiB | decode | **16.26 GiB/s** | 4.05 GiB/s | 15.76 GiB/s |
|
|
110
|
+
| | 1 MiB | encode | **19.29 GiB/s** | 4.79 GiB/s | 18.28 GiB/s |
|
|
111
|
+
| | 1 MiB | decode | **18.41 GiB/s** | 3.79 GiB/s | 16.93 GiB/s |
|
|
112
|
+
| | 32 MiB | encode | **10.29 GiB/s** | 3.08 GiB/s | 10.24 GiB/s |
|
|
113
|
+
| | 32 MiB | decode | **10.12 GiB/s** | 3.12 GiB/s | 9.83 GiB/s |
|
|
114
|
+
| URL-safe | 4 KiB | encode | **18.19 GiB/s** | 5.39 GiB/s | 16.98 GiB/s |
|
|
115
|
+
| | 4 KiB | decode | 14.28 GiB/s | 4.02 GiB/s | **15.77 GiB/s** |
|
|
116
|
+
| | 1 MiB | encode | **19.27 GiB/s** | 4.82 GiB/s | 18.32 GiB/s |
|
|
117
|
+
| | 1 MiB | decode | 16.28 GiB/s | 3.83 GiB/s | **16.99 GiB/s** |
|
|
118
|
+
| | 32 MiB | encode | 10.35 GiB/s | 3.11 GiB/s | **10.45 GiB/s** |
|
|
119
|
+
| | 32 MiB | decode | 9.83 GiB/s | 3.17 GiB/s | **9.85 GiB/s** |
|
|
120
|
+
|
|
121
|
+
## MurmurHash3: Rust
|
|
122
|
+
|
|
123
|
+
| Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
|
|
124
|
+
| --- | --- | ---: | ---: | ---: | ---: | ---: |
|
|
125
|
+
| x86 32-bit | 4 KiB | 4.01 GiB/s | 2.42 GiB/s | 3.83 GiB/s | n/a | **4.03 GiB/s** |
|
|
126
|
+
| | 1 MiB | **4.00 GiB/s** | 2.44 GiB/s | 3.79 GiB/s | n/a | 3.98 GiB/s |
|
|
127
|
+
| | 32 MiB | **3.99 GiB/s** | 2.42 GiB/s | 3.67 GiB/s | n/a | 3.83 GiB/s |
|
|
128
|
+
| x86 128-bit | 4 KiB | **8.93 GiB/s** | 4.66 GiB/s | 8.09 GiB/s | n/a | n/a |
|
|
129
|
+
| | 1 MiB | **9.24 GiB/s** | 4.72 GiB/s | 8.22 GiB/s | n/a | n/a |
|
|
130
|
+
| | 32 MiB | **9.13 GiB/s** | 4.56 GiB/s | 5.92 GiB/s | n/a | n/a |
|
|
131
|
+
| x64 128-bit | 4 KiB | **10.08 GiB/s** | 6.38 GiB/s | 8.75 GiB/s | 9.41 GiB/s | 8.93 GiB/s |
|
|
132
|
+
| | 1 MiB | **10.13 GiB/s** | 6.45 GiB/s | 8.90 GiB/s | 9.41 GiB/s | 8.90 GiB/s |
|
|
133
|
+
| | 32 MiB | **9.57 GiB/s** | 5.89 GiB/s | 6.42 GiB/s | 6.97 GiB/s | 6.44 GiB/s |
|
|
134
|
+
|
|
135
|
+
## Base64: Python
|
|
136
|
+
|
|
137
|
+
Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly into Rust without an input copy.
|
|
138
|
+
|
|
139
|
+
| Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
|
|
140
|
+
| --- | --- | --- | ---: | ---: | ---: |
|
|
141
|
+
| Standard | 4 KiB | encode | 13.74 GiB/s | 0.49 GiB/s | **14.05 GiB/s** |
|
|
142
|
+
| | 4 KiB | decode | **11.95 GiB/s** | 1.15 GiB/s | 8.47 GiB/s |
|
|
143
|
+
| | 1 MiB | encode | **3.70 GiB/s** | 0.47 GiB/s | 3.40 GiB/s |
|
|
144
|
+
| | 1 MiB | decode | **4.29 GiB/s** | 0.89 GiB/s | 4.24 GiB/s |
|
|
145
|
+
| | 32 MiB | encode | **3.81 GiB/s** | 0.47 GiB/s | 3.61 GiB/s |
|
|
146
|
+
| | 32 MiB | decode | **4.58 GiB/s** | 0.91 GiB/s | **4.58 GiB/s** |
|
|
147
|
+
| URL-safe | 4 KiB | encode | **13.54 GiB/s** | 0.41 GiB/s | 1.19 GiB/s |
|
|
148
|
+
| | 4 KiB | decode | **8.89 GiB/s** | 0.73 GiB/s | 1.56 GiB/s |
|
|
149
|
+
| | 1 MiB | encode | **3.70 GiB/s** | 0.36 GiB/s | 0.97 GiB/s |
|
|
150
|
+
| | 1 MiB | decode | **4.07 GiB/s** | 0.57 GiB/s | 1.56 GiB/s |
|
|
151
|
+
| | 32 MiB | encode | **3.88 GiB/s** | 0.37 GiB/s | 0.99 GiB/s |
|
|
152
|
+
| | 32 MiB | decode | **4.30 GiB/s** | 0.58 GiB/s | 1.67 GiB/s |
|
|
153
|
+
|
|
154
|
+
## MurmurHash3: Python
|
|
155
|
+
|
|
156
|
+
| Variant | API | Input | hashcodecs | `mmh3` |
|
|
157
|
+
| --- | --- | --- | ---: | ---: |
|
|
158
|
+
| x86 32-bit | one-shot | 4 KiB | **3.79 GiB/s** | 3.72 GiB/s |
|
|
159
|
+
| | | 1 MiB | **3.99 GiB/s** | 3.84 GiB/s |
|
|
160
|
+
| | | 32 MiB | **3.99 GiB/s** | 3.69 GiB/s |
|
|
161
|
+
| | incremental | 4 KiB | **3.60 GiB/s** | 3.59 GiB/s |
|
|
162
|
+
| | | 1 MiB | **4.00 GiB/s** | 3.84 GiB/s |
|
|
163
|
+
| | | 32 MiB | **3.99 GiB/s** | 3.80 GiB/s |
|
|
164
|
+
| x86 128-bit | one-shot | 4 KiB | 8.01 GiB/s | **8.15 GiB/s** |
|
|
165
|
+
| | | 1 MiB | **9.17 GiB/s** | 8.88 GiB/s |
|
|
166
|
+
| | | 32 MiB | **9.08 GiB/s** | 6.29 GiB/s |
|
|
167
|
+
| | incremental | 4 KiB | **7.07 GiB/s** | 0.79 GiB/s |
|
|
168
|
+
| | | 1 MiB | **9.21 GiB/s** | 0.80 GiB/s |
|
|
169
|
+
| | | 32 MiB | **9.17 GiB/s** | 0.79 GiB/s |
|
|
170
|
+
| x64 128-bit | one-shot | 4 KiB | 8.78 GiB/s | **9.48 GiB/s** |
|
|
171
|
+
| | | 1 MiB | 10.00 GiB/s | **10.26 GiB/s** |
|
|
172
|
+
| | | 32 MiB | **9.64 GiB/s** | 6.88 GiB/s |
|
|
173
|
+
| | incremental | 4 KiB | 7.75 GiB/s | **8.02 GiB/s** |
|
|
174
|
+
| | | 1 MiB | **10.09 GiB/s** | 9.36 GiB/s |
|
|
175
|
+
| | | 32 MiB | **9.62 GiB/s** | 7.58 GiB/s |
|
|
176
|
+
|
|
177
|
+
## SIMD References
|
|
178
|
+
|
|
179
|
+
The AVX2 block structure uses the 24-byte encode and 32-byte decode arrangement described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605). The AVX-512 VBMI path processes 48-byte encode and 64-byte decode blocks with cross-lane byte permutations, while AArch64 uses 48-byte NEON encoding and 64-byte NEON decoding blocks. The Rust 1.89 MSRV is the first stable release that exposes the required AVX-512 intrinsics.
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# hashcodecs
|
|
2
|
+
|
|
3
|
+
[](https://github.com/kozistr/hashcodecs-rs/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/hashcodecs/)
|
|
5
|
+
[](https://pypi.org/project/hashcodecs/)
|
|
6
|
+
[](https://github.com/kozistr/hashcodecs-rs#license)
|
|
7
|
+
[](https://pypi.org/project/hashcodecs/)
|
|
8
|
+
|
|
9
|
+
`hashcodecs` provides runtime-dispatched SIMD Base64 codecs and fast, reference-compatible MurmurHash3 functions for Rust and Python.
|
|
10
|
+
|
|
11
|
+
## Design
|
|
12
|
+
|
|
13
|
+
- Base64 is implemented in this crate with runtime AVX-512 VBMI, AVX2, SSE4.1, SSSE3, and AArch64 NEON dispatch. Portable wheels select the best available SIMD backend and fall back to scalar code otherwise.
|
|
14
|
+
- Rust callers can reuse their own output buffers through the `*_into` APIs. Every backend writes exactly the documented prefix; it never depends on spare capacity after the destination slice.
|
|
15
|
+
- Rust-owned results use the `mimalloc` global allocator.
|
|
16
|
+
- Every MurmurHash3 variant uses runtime-dispatched AVX2 or SSE4.1 pre-mixing where its measured crossover beats the scalar loop. The x64 128-bit AVX2 kernel also selects BMI2 rotations when available. Ordered state transitions remain reference-compatible, and every variant has a portable scalar fallback.
|
|
17
|
+
- The Python Base64 surface follows the familiar `base64` names. Use `import hashcodecs.base64 as base64` when replacing standard Base64 calls.
|
|
18
|
+
- Large Python calls release the GIL. Strict decoding and valid default-mode decoding write directly into the returned `bytes`; malformed default-mode input uses a CPython-compatible lenient fallback.
|
|
19
|
+
|
|
20
|
+
## Install
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
pip3 install hashcodecs
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Usage
|
|
27
|
+
|
|
28
|
+
### Rust
|
|
29
|
+
|
|
30
|
+
```rust
|
|
31
|
+
let encoded = hashcodecs::b64encode(b"hello");
|
|
32
|
+
assert_eq!(encoded, "aGVsbG8=");
|
|
33
|
+
|
|
34
|
+
let mut output = [0_u8; 8];
|
|
35
|
+
let written = hashcodecs::b64encode_into(b"hello", &mut output).unwrap();
|
|
36
|
+
assert_eq!(&output[..written], b"aGVsbG8=");
|
|
37
|
+
assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Python
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import hashcodecs.base64 as base64
|
|
44
|
+
from hashcodecs import murmur3_32, murmur3_x64_128
|
|
45
|
+
|
|
46
|
+
assert base64.b64encode(b"hello") == b"aGVsbG8="
|
|
47
|
+
assert base64.b64decode(b"aGVsbG8=") == b"hello"
|
|
48
|
+
assert murmur3_32(b"hello") == 0x248BFA47
|
|
49
|
+
|
|
50
|
+
hasher = murmur3_x64_128(seed=42)
|
|
51
|
+
hasher.update(b"hello")
|
|
52
|
+
assert hasher.hexdigest() == hasher.digest().hex()
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Build an installable wheel and source distribution with:
|
|
56
|
+
|
|
57
|
+
```sh
|
|
58
|
+
uv build
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
# Benchmark
|
|
62
|
+
|
|
63
|
+
Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
|
|
64
|
+
|
|
65
|
+
Conditions: clean builds, one pinned logical CPU, single-threaded execution, and output allocation included. Higher is better.
|
|
66
|
+
|
|
67
|
+
## Run Locally
|
|
68
|
+
|
|
69
|
+
Comparison crates are development-only dependencies and are not included in consumer builds.
|
|
70
|
+
|
|
71
|
+
```sh
|
|
72
|
+
cargo bench --bench base64
|
|
73
|
+
cargo bench --bench murmur3
|
|
74
|
+
uv sync --group benchmark --no-install-project
|
|
75
|
+
uv run --no-project python benchmarks/python_base64.py
|
|
76
|
+
uv run --no-project python benchmarks/python_murmur3.py
|
|
77
|
+
uv run --no-project python benchmarks/python_murmur3.py --incremental
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
To time only this implementation:
|
|
81
|
+
|
|
82
|
+
```sh
|
|
83
|
+
cargo bench --bench base64 -- hashcodecs
|
|
84
|
+
cargo bench --bench murmur3 -- hashcodecs
|
|
85
|
+
uv run --no-project python benchmarks/python_base64.py --hashcodecs-only
|
|
86
|
+
uv run --no-project python benchmarks/python_murmur3.py --hashcodecs-only
|
|
87
|
+
uv run --no-project python benchmarks/python_murmur3.py --incremental --hashcodecs-only
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Base64: Rust
|
|
91
|
+
|
|
92
|
+
| Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
|
|
93
|
+
| --- | --- | --- | ---: | ---: | ---: |
|
|
94
|
+
| Standard | 4 KiB | encode | **18.17 GiB/s** | 5.40 GiB/s | 16.97 GiB/s |
|
|
95
|
+
| | 4 KiB | decode | **16.26 GiB/s** | 4.05 GiB/s | 15.76 GiB/s |
|
|
96
|
+
| | 1 MiB | encode | **19.29 GiB/s** | 4.79 GiB/s | 18.28 GiB/s |
|
|
97
|
+
| | 1 MiB | decode | **18.41 GiB/s** | 3.79 GiB/s | 16.93 GiB/s |
|
|
98
|
+
| | 32 MiB | encode | **10.29 GiB/s** | 3.08 GiB/s | 10.24 GiB/s |
|
|
99
|
+
| | 32 MiB | decode | **10.12 GiB/s** | 3.12 GiB/s | 9.83 GiB/s |
|
|
100
|
+
| URL-safe | 4 KiB | encode | **18.19 GiB/s** | 5.39 GiB/s | 16.98 GiB/s |
|
|
101
|
+
| | 4 KiB | decode | 14.28 GiB/s | 4.02 GiB/s | **15.77 GiB/s** |
|
|
102
|
+
| | 1 MiB | encode | **19.27 GiB/s** | 4.82 GiB/s | 18.32 GiB/s |
|
|
103
|
+
| | 1 MiB | decode | 16.28 GiB/s | 3.83 GiB/s | **16.99 GiB/s** |
|
|
104
|
+
| | 32 MiB | encode | 10.35 GiB/s | 3.11 GiB/s | **10.45 GiB/s** |
|
|
105
|
+
| | 32 MiB | decode | 9.83 GiB/s | 3.17 GiB/s | **9.85 GiB/s** |
|
|
106
|
+
|
|
107
|
+
## MurmurHash3: Rust
|
|
108
|
+
|
|
109
|
+
| Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
|
|
110
|
+
| --- | --- | ---: | ---: | ---: | ---: | ---: |
|
|
111
|
+
| x86 32-bit | 4 KiB | 4.01 GiB/s | 2.42 GiB/s | 3.83 GiB/s | n/a | **4.03 GiB/s** |
|
|
112
|
+
| | 1 MiB | **4.00 GiB/s** | 2.44 GiB/s | 3.79 GiB/s | n/a | 3.98 GiB/s |
|
|
113
|
+
| | 32 MiB | **3.99 GiB/s** | 2.42 GiB/s | 3.67 GiB/s | n/a | 3.83 GiB/s |
|
|
114
|
+
| x86 128-bit | 4 KiB | **8.93 GiB/s** | 4.66 GiB/s | 8.09 GiB/s | n/a | n/a |
|
|
115
|
+
| | 1 MiB | **9.24 GiB/s** | 4.72 GiB/s | 8.22 GiB/s | n/a | n/a |
|
|
116
|
+
| | 32 MiB | **9.13 GiB/s** | 4.56 GiB/s | 5.92 GiB/s | n/a | n/a |
|
|
117
|
+
| x64 128-bit | 4 KiB | **10.08 GiB/s** | 6.38 GiB/s | 8.75 GiB/s | 9.41 GiB/s | 8.93 GiB/s |
|
|
118
|
+
| | 1 MiB | **10.13 GiB/s** | 6.45 GiB/s | 8.90 GiB/s | 9.41 GiB/s | 8.90 GiB/s |
|
|
119
|
+
| | 32 MiB | **9.57 GiB/s** | 5.89 GiB/s | 6.42 GiB/s | 6.97 GiB/s | 6.44 GiB/s |
|
|
120
|
+
|
|
121
|
+
## Base64: Python
|
|
122
|
+
|
|
123
|
+
Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly into Rust without an input copy.
|
|
124
|
+
|
|
125
|
+
| Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
|
|
126
|
+
| --- | --- | --- | ---: | ---: | ---: |
|
|
127
|
+
| Standard | 4 KiB | encode | 13.74 GiB/s | 0.49 GiB/s | **14.05 GiB/s** |
|
|
128
|
+
| | 4 KiB | decode | **11.95 GiB/s** | 1.15 GiB/s | 8.47 GiB/s |
|
|
129
|
+
| | 1 MiB | encode | **3.70 GiB/s** | 0.47 GiB/s | 3.40 GiB/s |
|
|
130
|
+
| | 1 MiB | decode | **4.29 GiB/s** | 0.89 GiB/s | 4.24 GiB/s |
|
|
131
|
+
| | 32 MiB | encode | **3.81 GiB/s** | 0.47 GiB/s | 3.61 GiB/s |
|
|
132
|
+
| | 32 MiB | decode | **4.58 GiB/s** | 0.91 GiB/s | **4.58 GiB/s** |
|
|
133
|
+
| URL-safe | 4 KiB | encode | **13.54 GiB/s** | 0.41 GiB/s | 1.19 GiB/s |
|
|
134
|
+
| | 4 KiB | decode | **8.89 GiB/s** | 0.73 GiB/s | 1.56 GiB/s |
|
|
135
|
+
| | 1 MiB | encode | **3.70 GiB/s** | 0.36 GiB/s | 0.97 GiB/s |
|
|
136
|
+
| | 1 MiB | decode | **4.07 GiB/s** | 0.57 GiB/s | 1.56 GiB/s |
|
|
137
|
+
| | 32 MiB | encode | **3.88 GiB/s** | 0.37 GiB/s | 0.99 GiB/s |
|
|
138
|
+
| | 32 MiB | decode | **4.30 GiB/s** | 0.58 GiB/s | 1.67 GiB/s |
|
|
139
|
+
|
|
140
|
+
## MurmurHash3: Python
|
|
141
|
+
|
|
142
|
+
| Variant | API | Input | hashcodecs | `mmh3` |
|
|
143
|
+
| --- | --- | --- | ---: | ---: |
|
|
144
|
+
| x86 32-bit | one-shot | 4 KiB | **3.79 GiB/s** | 3.72 GiB/s |
|
|
145
|
+
| | | 1 MiB | **3.99 GiB/s** | 3.84 GiB/s |
|
|
146
|
+
| | | 32 MiB | **3.99 GiB/s** | 3.69 GiB/s |
|
|
147
|
+
| | incremental | 4 KiB | **3.60 GiB/s** | 3.59 GiB/s |
|
|
148
|
+
| | | 1 MiB | **4.00 GiB/s** | 3.84 GiB/s |
|
|
149
|
+
| | | 32 MiB | **3.99 GiB/s** | 3.80 GiB/s |
|
|
150
|
+
| x86 128-bit | one-shot | 4 KiB | 8.01 GiB/s | **8.15 GiB/s** |
|
|
151
|
+
| | | 1 MiB | **9.17 GiB/s** | 8.88 GiB/s |
|
|
152
|
+
| | | 32 MiB | **9.08 GiB/s** | 6.29 GiB/s |
|
|
153
|
+
| | incremental | 4 KiB | **7.07 GiB/s** | 0.79 GiB/s |
|
|
154
|
+
| | | 1 MiB | **9.21 GiB/s** | 0.80 GiB/s |
|
|
155
|
+
| | | 32 MiB | **9.17 GiB/s** | 0.79 GiB/s |
|
|
156
|
+
| x64 128-bit | one-shot | 4 KiB | 8.78 GiB/s | **9.48 GiB/s** |
|
|
157
|
+
| | | 1 MiB | 10.00 GiB/s | **10.26 GiB/s** |
|
|
158
|
+
| | | 32 MiB | **9.64 GiB/s** | 6.88 GiB/s |
|
|
159
|
+
| | incremental | 4 KiB | 7.75 GiB/s | **8.02 GiB/s** |
|
|
160
|
+
| | | 1 MiB | **10.09 GiB/s** | 9.36 GiB/s |
|
|
161
|
+
| | | 32 MiB | **9.62 GiB/s** | 7.58 GiB/s |
|
|
162
|
+
|
|
163
|
+
## SIMD References
|
|
164
|
+
|
|
165
|
+
The AVX2 block structure uses the 24-byte encode and 32-byte decode arrangement described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605). The AVX-512 VBMI path processes 48-byte encode and 64-byte decode blocks with cross-lane byte permutations, while AArch64 uses 48-byte NEON encoding and 64-byte NEON decoding blocks. The Rust 1.89 MSRV is the first stable release that exposes the required AVX-512 intrinsics.
|
|
@@ -6,8 +6,8 @@ use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_m
|
|
|
6
6
|
|
|
7
7
|
mod support;
|
|
8
8
|
|
|
9
|
-
const
|
|
10
|
-
const
|
|
9
|
+
const SIZES: [usize; 3] = [4 * 1024, 1024 * 1024, 32 * 1024 * 1024];
|
|
10
|
+
const SAMPLE_SIZE: usize = 50;
|
|
11
11
|
|
|
12
12
|
fn data(size: usize) -> Vec<u8> {
|
|
13
13
|
(0..size)
|
|
@@ -23,10 +23,6 @@ macro_rules! benchmark {
|
|
|
23
23
|
};
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
fn sample_size(size: usize) -> usize {
|
|
27
|
-
if size == LARGE_SIZE { 10 } else { 30 }
|
|
28
|
-
}
|
|
29
|
-
|
|
30
26
|
fn base64(c: &mut Criterion) {
|
|
31
27
|
support::pin_to_one_cpu();
|
|
32
28
|
standard_encode(c);
|
|
@@ -46,7 +42,7 @@ fn standard_encode(c: &mut Criterion) {
|
|
|
46
42
|
);
|
|
47
43
|
assert_eq!(base64_turbo::STANDARD.encode(&input), expected);
|
|
48
44
|
|
|
49
|
-
group.sample_size(
|
|
45
|
+
group.sample_size(SAMPLE_SIZE);
|
|
50
46
|
group.throughput(Throughput::Bytes(size as u64));
|
|
51
47
|
benchmark!(group, size, &input, "hashcodecs", hashcodecs::b64encode);
|
|
52
48
|
benchmark!(group, size, &input, "base64", |input: &[u8]| {
|
|
@@ -70,7 +66,7 @@ fn urlsafe_encode(c: &mut Criterion) {
|
|
|
70
66
|
);
|
|
71
67
|
assert_eq!(base64_turbo::URL_SAFE.encode(&input), expected);
|
|
72
68
|
|
|
73
|
-
group.sample_size(
|
|
69
|
+
group.sample_size(SAMPLE_SIZE);
|
|
74
70
|
group.throughput(Throughput::Bytes(size as u64));
|
|
75
71
|
benchmark!(
|
|
76
72
|
group,
|
|
@@ -102,7 +98,7 @@ fn standard_decode(c: &mut Criterion) {
|
|
|
102
98
|
);
|
|
103
99
|
assert_eq!(base64_turbo::STANDARD.decode(&input).unwrap(), expected);
|
|
104
100
|
|
|
105
|
-
group.sample_size(
|
|
101
|
+
group.sample_size(SAMPLE_SIZE);
|
|
106
102
|
group.throughput(Throughput::Bytes(size as u64));
|
|
107
103
|
benchmark!(
|
|
108
104
|
group,
|
|
@@ -140,7 +136,7 @@ fn urlsafe_decode(c: &mut Criterion) {
|
|
|
140
136
|
);
|
|
141
137
|
assert_eq!(base64_turbo::URL_SAFE.decode(&input).unwrap(), expected);
|
|
142
138
|
|
|
143
|
-
group.sample_size(
|
|
139
|
+
group.sample_size(SAMPLE_SIZE);
|
|
144
140
|
group.throughput(Throughput::Bytes(size as u64));
|
|
145
141
|
benchmark!(
|
|
146
142
|
group,
|
|
@@ -6,8 +6,8 @@ use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_m
|
|
|
6
6
|
|
|
7
7
|
mod support;
|
|
8
8
|
|
|
9
|
-
const
|
|
10
|
-
const
|
|
9
|
+
const SIZES: [usize; 3] = [4 * 1024, 1024 * 1024, 32 * 1024 * 1024];
|
|
10
|
+
const SAMPLE_SIZE: usize = 50;
|
|
11
11
|
|
|
12
12
|
fn data(size: usize) -> Vec<u8> {
|
|
13
13
|
(0..size)
|
|
@@ -52,7 +52,7 @@ fn x86_32(c: &mut Criterion) {
|
|
|
52
52
|
);
|
|
53
53
|
assert_eq!(murmurs::murmur3_x86_32(&input, 42), expected);
|
|
54
54
|
assert_eq!(mm3h::murmurhash3_32_with_seed(&input, 42), expected);
|
|
55
|
-
group.sample_size(
|
|
55
|
+
group.sample_size(SAMPLE_SIZE);
|
|
56
56
|
group.throughput(Throughput::Bytes(size as u64));
|
|
57
57
|
benchmark!(group, size, &input, "hashcodecs", |input: &[u8]| {
|
|
58
58
|
hashcodecs::murmur3_x86_32(input, 42)
|
|
@@ -80,7 +80,7 @@ fn x86_128(c: &mut Criterion) {
|
|
|
80
80
|
x86_128_as_u128(expected)
|
|
81
81
|
);
|
|
82
82
|
assert_eq!(murmurs::murmur3_x86_128(&input, 42), expected);
|
|
83
|
-
group.sample_size(
|
|
83
|
+
group.sample_size(SAMPLE_SIZE);
|
|
84
84
|
group.throughput(Throughput::Bytes(size as u64));
|
|
85
85
|
benchmark!(group, size, &input, "hashcodecs", |input: &[u8]| {
|
|
86
86
|
hashcodecs::murmur3_x86_128(input, 42)
|
|
@@ -108,7 +108,7 @@ fn x64_128(c: &mut Criterion) {
|
|
|
108
108
|
assert_eq!(murmurs::murmur3_x64_128(&input, 42), expected);
|
|
109
109
|
assert_eq!(fastmurmur3::murmur3_x64_128(&input, 42), expected_u128);
|
|
110
110
|
assert_eq!(mm3h::murmurhash3_128_with_seed(&input, 42), expected_u128);
|
|
111
|
-
group.sample_size(
|
|
111
|
+
group.sample_size(SAMPLE_SIZE);
|
|
112
112
|
group.throughput(Throughput::Bytes(size as u64));
|
|
113
113
|
benchmark!(group, size, &input, "hashcodecs", |input: &[u8]| {
|
|
114
114
|
hashcodecs::murmur3_x64_128(input, 42)
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "hashcodecs"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "SIMD-accelerated Base64 and MurmurHash3 codecs"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -21,9 +21,12 @@ classifiers = [
|
|
|
21
21
|
dev = [
|
|
22
22
|
"pytest>=9,<10",
|
|
23
23
|
"pytest-cov>=7,<8",
|
|
24
|
-
"pybase64>=1.4.3,<2",
|
|
25
24
|
"ruff>=0.16,<0.17",
|
|
26
25
|
]
|
|
26
|
+
benchmark = [
|
|
27
|
+
"mmh3>=5.2.1,<6",
|
|
28
|
+
"pybase64>=1.4.3,<2",
|
|
29
|
+
]
|
|
27
30
|
|
|
28
31
|
[tool.hatch.build.targets.wheel]
|
|
29
32
|
packages = ["python/hashcodecs"]
|
|
@@ -37,11 +40,10 @@ include = [
|
|
|
37
40
|
"/LICENSE*",
|
|
38
41
|
"/README.md",
|
|
39
42
|
"/benches",
|
|
40
|
-
"/benchmarks",
|
|
41
43
|
"/python",
|
|
42
44
|
"/src",
|
|
43
|
-
"/tests",
|
|
44
45
|
]
|
|
46
|
+
exclude = ["/src/base64/tests.rs"]
|
|
45
47
|
|
|
46
48
|
[tool.hatch.build.hooks.custom]
|
|
47
49
|
path = "hatch_build.py"
|
|
@@ -50,7 +52,7 @@ path = "hatch_build.py"
|
|
|
50
52
|
build = "cp310-*"
|
|
51
53
|
build-frontend = "uv"
|
|
52
54
|
skip = "*-musllinux*"
|
|
53
|
-
test-command = "python -c \"import hashcodecs; assert hashcodecs.b64decode(hashcodecs.b64encode(b'
|
|
55
|
+
test-command = "python -c \"import hashcodecs; assert hashcodecs.b64decode(hashcodecs.b64encode(b'hashcodecs')) == b'hashcodecs'\""
|
|
54
56
|
|
|
55
57
|
[tool.cibuildwheel.linux]
|
|
56
58
|
before-all = "curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal"
|
|
@@ -1,13 +1,23 @@
|
|
|
1
1
|
"""SIMD-accelerated Base64 and MurmurHash3 functions."""
|
|
2
2
|
|
|
3
3
|
from .base64 import b64decode, b64encode, standard_b64decode, standard_b64encode, urlsafe_b64decode, urlsafe_b64encode
|
|
4
|
-
from .murmur3 import
|
|
4
|
+
from .murmur3 import (
|
|
5
|
+
murmur3_32,
|
|
6
|
+
murmur3_x64_128,
|
|
7
|
+
murmur3_x64_128_digest,
|
|
8
|
+
murmur3_x86_32,
|
|
9
|
+
murmur3_x86_128,
|
|
10
|
+
murmur3_x86_128_digest,
|
|
11
|
+
)
|
|
5
12
|
|
|
6
13
|
__all__ = [
|
|
7
14
|
"b64decode",
|
|
8
15
|
"b64encode",
|
|
9
16
|
"murmur3_32",
|
|
17
|
+
"murmur3_x64_128",
|
|
10
18
|
"murmur3_x64_128_digest",
|
|
19
|
+
"murmur3_x86_32",
|
|
20
|
+
"murmur3_x86_128",
|
|
11
21
|
"murmur3_x86_128_digest",
|
|
12
22
|
"standard_b64decode",
|
|
13
23
|
"standard_b64encode",
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from _typeshed import ReadableBuffer
|
|
2
|
+
|
|
3
|
+
def b64encode(s: ReadableBuffer, altchars: ReadableBuffer | None = None) -> bytes: ...
|
|
4
|
+
def b64decode(
|
|
5
|
+
s: str | ReadableBuffer,
|
|
6
|
+
altchars: str | ReadableBuffer | None = None,
|
|
7
|
+
validate: bool = False,
|
|
8
|
+
) -> bytes: ...
|
|
9
|
+
def murmur3_32(s: ReadableBuffer, seed: int = 0) -> int: ...
|
|
10
|
+
def murmur3_x86_128_digest(s: ReadableBuffer, seed: int = 0) -> bytes: ...
|
|
11
|
+
def murmur3_x64_128_digest(s: ReadableBuffer, seed: int = 0) -> bytes: ...
|
|
12
|
+
|
|
13
|
+
class murmur3_x86_32:
|
|
14
|
+
def __init__(self, data: ReadableBuffer | None = None, seed: int = 0) -> None: ...
|
|
15
|
+
def update(self, data: ReadableBuffer) -> None: ...
|
|
16
|
+
def digest(self) -> bytes: ...
|
|
17
|
+
def hexdigest(self) -> str: ...
|
|
18
|
+
def copy(self) -> murmur3_x86_32: ...
|
|
19
|
+
@property
|
|
20
|
+
def digest_size(self) -> int: ...
|
|
21
|
+
@property
|
|
22
|
+
def block_size(self) -> int: ...
|
|
23
|
+
@property
|
|
24
|
+
def name(self) -> str: ...
|
|
25
|
+
|
|
26
|
+
class murmur3_x86_128:
|
|
27
|
+
def __init__(self, data: ReadableBuffer | None = None, seed: int = 0) -> None: ...
|
|
28
|
+
def update(self, data: ReadableBuffer) -> None: ...
|
|
29
|
+
def digest(self) -> bytes: ...
|
|
30
|
+
def hexdigest(self) -> str: ...
|
|
31
|
+
def copy(self) -> murmur3_x86_128: ...
|
|
32
|
+
@property
|
|
33
|
+
def digest_size(self) -> int: ...
|
|
34
|
+
@property
|
|
35
|
+
def block_size(self) -> int: ...
|
|
36
|
+
@property
|
|
37
|
+
def name(self) -> str: ...
|
|
38
|
+
|
|
39
|
+
class murmur3_x64_128:
|
|
40
|
+
def __init__(self, data: ReadableBuffer | None = None, seed: int = 0) -> None: ...
|
|
41
|
+
def update(self, data: ReadableBuffer) -> None: ...
|
|
42
|
+
def digest(self) -> bytes: ...
|
|
43
|
+
def hexdigest(self) -> str: ...
|
|
44
|
+
def copy(self) -> murmur3_x64_128: ...
|
|
45
|
+
@property
|
|
46
|
+
def digest_size(self) -> int: ...
|
|
47
|
+
@property
|
|
48
|
+
def block_size(self) -> int: ...
|
|
49
|
+
@property
|
|
50
|
+
def name(self) -> str: ...
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Fast one-shot and incremental MurmurHash3 functions."""
|
|
2
|
+
|
|
3
|
+
from ._hashcodecs import (
|
|
4
|
+
murmur3_32,
|
|
5
|
+
murmur3_x64_128,
|
|
6
|
+
murmur3_x64_128_digest,
|
|
7
|
+
murmur3_x86_32,
|
|
8
|
+
murmur3_x86_128,
|
|
9
|
+
murmur3_x86_128_digest,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"murmur3_32",
|
|
14
|
+
"murmur3_x64_128",
|
|
15
|
+
"murmur3_x64_128_digest",
|
|
16
|
+
"murmur3_x86_32",
|
|
17
|
+
"murmur3_x86_128",
|
|
18
|
+
"murmur3_x86_128_digest",
|
|
19
|
+
]
|