hashcodecs 0.1.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/Cargo.lock +1 -1
  2. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/Cargo.toml +2 -2
  3. hashcodecs-0.3.0/PKG-INFO +202 -0
  4. hashcodecs-0.3.0/README.md +188 -0
  5. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/benches/base64.rs +6 -10
  6. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/benches/murmur3.rs +5 -5
  7. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/build.rs +2 -0
  8. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/hatch_build.py +19 -19
  9. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/pyproject.toml +21 -5
  10. hashcodecs-0.3.0/python/hashcodecs/__init__.py +45 -0
  11. hashcodecs-0.3.0/python/hashcodecs/_hashcodecs.pyi +57 -0
  12. hashcodecs-0.3.0/python/hashcodecs/base64.py +59 -0
  13. hashcodecs-0.3.0/python/hashcodecs/base64.pyi +23 -0
  14. hashcodecs-0.3.0/python/hashcodecs/murmur3.py +19 -0
  15. hashcodecs-0.3.0/src/base64/aarch64.rs +293 -0
  16. hashcodecs-0.3.0/src/base64/dispatch.rs +310 -0
  17. hashcodecs-0.3.0/src/base64/x86.rs +651 -0
  18. hashcodecs-0.3.0/src/base64/x86_avx512.rs +232 -0
  19. hashcodecs-0.3.0/src/base64.rs +555 -0
  20. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/src/lib.rs +5 -2
  21. hashcodecs-0.3.0/src/murmur3/dispatch.rs +47 -0
  22. hashcodecs-0.3.0/src/murmur3/x86.rs +472 -0
  23. hashcodecs-0.3.0/src/murmur3.rs +883 -0
  24. hashcodecs-0.1.0/src/python.rs → hashcodecs-0.3.0/src/python/base64.rs +124 -153
  25. hashcodecs-0.3.0/src/python/buffer.rs +88 -0
  26. hashcodecs-0.3.0/src/python/murmur3.rs +275 -0
  27. hashcodecs-0.3.0/src/python.rs +29 -0
  28. hashcodecs-0.1.0/PKG-INFO +0 -141
  29. hashcodecs-0.1.0/README.md +0 -127
  30. hashcodecs-0.1.0/benchmarks/python_base64.py +0 -200
  31. hashcodecs-0.1.0/python/hashcodecs/__init__.py +0 -16
  32. hashcodecs-0.1.0/python/hashcodecs/_hashcodecs.pyi +0 -11
  33. hashcodecs-0.1.0/python/hashcodecs/base64.py +0 -33
  34. hashcodecs-0.1.0/python/hashcodecs/base64.pyi +0 -12
  35. hashcodecs-0.1.0/python/hashcodecs/murmur3.py +0 -9
  36. hashcodecs-0.1.0/src/base64.rs +0 -1603
  37. hashcodecs-0.1.0/src/murmur3.rs +0 -337
  38. hashcodecs-0.1.0/tests/test_python_api.py +0 -156
  39. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/.gitignore +0 -0
  40. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/LICENSE +0 -0
  41. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/LICENSE-MIT +0 -0
  42. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/benches/support/mod.rs +0 -0
  43. {hashcodecs-0.1.0 → hashcodecs-0.3.0}/python/hashcodecs/py.typed +0 -0
{hashcodecs-0.1.0 → hashcodecs-0.3.0}/Cargo.lock RENAMED
@@ -194,7 +194,7 @@ dependencies = [
194
194
 
195
195
  [[package]]
196
196
  name = "hashcodecs"
197
- version = "0.1.0"
197
+ version = "0.3.0"
198
198
  dependencies = [
199
199
  "base64",
200
200
  "base64-turbo",
{hashcodecs-0.1.0 → hashcodecs-0.3.0}/Cargo.toml RENAMED
@@ -1,8 +1,8 @@
1
1
  [package]
2
2
  name = "hashcodecs"
3
- version = "0.1.0"
3
+ version = "0.3.0"
4
4
  edition = "2024"
5
- rust-version = "1.85"
5
+ rust-version = "1.89"
6
6
  description = "SIMD-accelerated Base64 codecs and fast MurmurHash3 implementations"
7
7
  license = "MIT OR Apache-2.0"
8
8
  repository = "https://github.com/kozistr/hashcodecs-rs"
hashcodecs-0.3.0/PKG-INFO ADDED
@@ -0,0 +1,202 @@
1
+ Metadata-Version: 2.4
2
+ Name: hashcodecs
3
+ Version: 0.3.0
4
+ Summary: SIMD-accelerated Base64 and MurmurHash3 codecs
5
+ License-Expression: MIT OR Apache-2.0
6
+ License-File: LICENSE
7
+ License-File: LICENSE-MIT
8
+ Classifier: Operating System :: OS Independent
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Rust
11
+ Classifier: Topic :: Software Development :: Libraries
12
+ Requires-Python: >=3.10
13
+ Description-Content-Type: text/markdown
14
+
15
+ # hashcodecs
16
+
17
+ [![CI](https://img.shields.io/github/actions/workflow/status/kozistr/hashcodecs-rs/ci.yml?branch=main&style=for-the-badge&logo=github)](https://github.com/kozistr/hashcodecs-rs/actions/workflows/ci.yml)
18
+ [![PyPI](https://img.shields.io/pypi/v/hashcodecs?style=for-the-badge&logo=pypi)](https://pypi.org/project/hashcodecs/)
19
+ [![Python](https://img.shields.io/pypi/pyversions/hashcodecs?style=for-the-badge&logo=python)](https://pypi.org/project/hashcodecs/)
20
+ [![License](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-brightgreen?style=for-the-badge)](https://github.com/kozistr/hashcodecs-rs#license)
21
+ [![Downloads](https://img.shields.io/pypi/dm/hashcodecs?style=for-the-badge&label=downloads)](https://pypi.org/project/hashcodecs/)
22
+
23
+ `hashcodecs` provides runtime-dispatched SIMD Base64 codecs and fast, reference-compatible MurmurHash3 functions for Rust and Python.
24
+
25
+ ## Design
26
+
27
+ - Base64 is implemented in this crate with runtime AVX-512 VBMI, AVX2, SSE4.1, SSSE3, and AArch64 NEON dispatch. Portable wheels select the best available SIMD backend and fall back to scalar code otherwise.
28
+ - Rust callers can reuse their own output buffers through the `*_into` APIs. Every backend writes exactly the documented prefix; it never depends on spare capacity after the destination slice.
29
+ - Rust-owned results use the `mimalloc` global allocator.
30
+ - Every MurmurHash3 variant uses runtime-dispatched AVX2 or SSE4.1 pre-mixing where its measured crossover beats the scalar loop. The x64 128-bit AVX2 kernel also selects BMI2 rotations when available. Ordered state transitions remain reference-compatible, and every variant has a portable scalar fallback.
31
+ - The Python Base64 surface follows the familiar `base64` names. Use `import hashcodecs.base64 as base64` when replacing standard Base64 calls.
32
+ - Python callers can reuse a `bytearray` through the `*_into` APIs when returned `bytes` ownership is unnecessary.
33
+ - Large Python calls release the GIL. Strict decoding and valid default-mode decoding write directly into the returned `bytes`; malformed default-mode input uses a CPython-compatible lenient fallback.
34
+
35
+ ## Install
36
+
37
+ ```
38
+ pip3 install hashcodecs
39
+ ```
40
+
41
+ ## Usage
42
+
43
+ ### Rust
44
+
45
+ ```rust
46
+ let encoded = hashcodecs::b64encode(b"hello");
47
+ assert_eq!(encoded, "aGVsbG8=");
48
+
49
+ let mut output = [0_u8; 8];
50
+ let written = hashcodecs::b64encode_into(b"hello", &mut output).unwrap();
51
+ assert_eq!(&output[..written], b"aGVsbG8=");
52
+ assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
53
+ ```
54
+
55
+ ### Python
56
+
57
+ ```python
58
+ import hashcodecs.base64 as base64
59
+ from hashcodecs import murmur3_32, murmur3_x64_128
60
+
61
+ assert base64.b64encode(b'hello') == b'aGVsbG8='
62
+ assert base64.b64decode(b'aGVsbG8=') == b'hello'
63
+ assert murmur3_32(b'hello') == 0x248BFA47
64
+
65
+ output = bytearray(8)
66
+ written = base64.b64encode_into(b'hello', output)
67
+ assert output[:written] == b'aGVsbG8='
68
+
69
+ hasher = murmur3_x64_128(seed=42)
70
+ hasher.update(b'hello')
71
+ assert hasher.hexdigest() == hasher.digest().hex()
72
+ ```
73
+
74
+ Build an installable wheel and source distribution with:
75
+
76
+ ```sh
77
+ uv build
78
+ ```
79
+
80
+ # Benchmark
81
+
82
+ Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
83
+
84
+ Conditions: clean builds, one pinned logical CPU, single-threaded execution, 50 Rust samples, and nine Python samples. Returned-output allocation is included except in the reusable-buffer table. Higher is better.
85
+
86
+ ## Run Locally
87
+
88
+ Comparison crates are development-only dependencies and are not included in consumer builds.
89
+
90
+ ```sh
91
+ cargo bench --bench base64
92
+ cargo bench --bench murmur3
93
+ uv sync --group benchmark --no-install-project
94
+ uv run --no-project python benchmarks/python_base64.py
95
+ uv run --no-project python benchmarks/python_base64.py --into
96
+ uv run --no-project python benchmarks/python_murmur3.py
97
+ uv run --no-project python benchmarks/python_murmur3.py --incremental
98
+ ```
99
+
100
+ To time only this implementation:
101
+
102
+ ```sh
103
+ cargo bench --bench base64 -- hashcodecs
104
+ cargo bench --bench murmur3 -- hashcodecs
105
+ uv run --no-project python benchmarks/python_base64.py --hashcodecs-only
106
+ uv run --no-project python benchmarks/python_murmur3.py --hashcodecs-only
107
+ uv run --no-project python benchmarks/python_murmur3.py --incremental --hashcodecs-only
108
+ ```
109
+
110
+ ## Base64: Rust
111
+
112
+ | Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
113
+ | --- | --- | --- | ---: | ---: | ---: |
114
+ | Standard | 4 KiB | encode | **18.11 GiB/s** | 5.39 GiB/s | 16.94 GiB/s |
115
+ | | 4 KiB | decode | **24.70 GiB/s** | 4.06 GiB/s | 15.73 GiB/s |
116
+ | | 1 MiB | encode | **19.20 GiB/s** | 4.81 GiB/s | 18.19 GiB/s |
117
+ | | 1 MiB | decode | **28.72 GiB/s** | 3.80 GiB/s | 16.66 GiB/s |
118
+ | | 32 MiB | encode | **10.32 GiB/s** | 2.95 GiB/s | 9.97 GiB/s |
119
+ | | 32 MiB | decode | **10.56 GiB/s** | 3.04 GiB/s | 9.70 GiB/s |
120
+ | URL-safe | 4 KiB | encode | **17.95 GiB/s** | 5.40 GiB/s | 16.90 GiB/s |
121
+ | | 4 KiB | decode | **23.39 GiB/s** | 4.05 GiB/s | 15.75 GiB/s |
122
+ | | 1 MiB | encode | **19.19 GiB/s** | 4.80 GiB/s | 18.17 GiB/s |
123
+ | | 1 MiB | decode | **27.19 GiB/s** | 3.79 GiB/s | 16.76 GiB/s |
124
+ | | 32 MiB | encode | **10.33 GiB/s** | 2.98 GiB/s | 10.16 GiB/s |
125
+ | | 32 MiB | decode | **10.80 GiB/s** | 3.10 GiB/s | 9.36 GiB/s |
126
+
127
+ ## MurmurHash3: Rust
128
+
129
+ | Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
130
+ | --- | --- | ---: | ---: | ---: | ---: | ---: |
131
+ | x86 32-bit | 4 KiB | **4.03 GiB/s** | 2.44 GiB/s | 3.82 GiB/s | n/a | 4.02 GiB/s |
132
+ | | 1 MiB | **3.99 GiB/s** | 2.43 GiB/s | 3.79 GiB/s | n/a | 3.97 GiB/s |
133
+ | | 32 MiB | **3.98 GiB/s** | 2.42 GiB/s | 3.65 GiB/s | n/a | 3.81 GiB/s |
134
+ | x86 128-bit | 4 KiB | **8.90 GiB/s** | 4.66 GiB/s | 8.07 GiB/s | n/a | n/a |
135
+ | | 1 MiB | **9.21 GiB/s** | 4.71 GiB/s | 8.22 GiB/s | n/a | n/a |
136
+ | | 32 MiB | **9.12 GiB/s** | 4.55 GiB/s | 5.73 GiB/s | n/a | n/a |
137
+ | x64 128-bit | 4 KiB | **10.08 GiB/s** | 6.38 GiB/s | 8.85 GiB/s | 9.41 GiB/s | 8.91 GiB/s |
138
+ | | 1 MiB | **10.11 GiB/s** | 6.44 GiB/s | 8.87 GiB/s | 9.37 GiB/s | 8.88 GiB/s |
139
+ | | 32 MiB | **9.54 GiB/s** | 5.83 GiB/s | 6.39 GiB/s | 6.99 GiB/s | 6.41 GiB/s |
140
+
141
+ ## Base64: Python
142
+
143
+ Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly into Rust without an input copy.
144
+
145
+ | Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
146
+ | --- | --- | --- | ---: | ---: | ---: |
147
+ | Standard | 4 KiB | encode | 13.73 GiB/s | 0.49 GiB/s | **14.28 GiB/s** |
148
+ | | 4 KiB | decode | **15.64 GiB/s** | 1.15 GiB/s | 8.44 GiB/s |
149
+ | | 1 MiB | encode | **3.68 GiB/s** | 0.47 GiB/s | 3.32 GiB/s |
150
+ | | 1 MiB | decode | 3.64 GiB/s | 0.89 GiB/s | **4.13 GiB/s** |
151
+ | | 32 MiB | encode | **3.75 GiB/s** | 0.47 GiB/s | 3.51 GiB/s |
152
+ | | 32 MiB | decode | 3.81 GiB/s | 0.90 GiB/s | **4.42 GiB/s** |
153
+ | URL-safe | 4 KiB | encode | **13.68 GiB/s** | 0.41 GiB/s | 1.19 GiB/s |
154
+ | | 4 KiB | decode | **12.66 GiB/s** | 0.74 GiB/s | 1.56 GiB/s |
155
+ | | 1 MiB | encode | **3.63 GiB/s** | 0.36 GiB/s | 0.97 GiB/s |
156
+ | | 1 MiB | decode | **4.06 GiB/s** | 0.57 GiB/s | 1.55 GiB/s |
157
+ | | 32 MiB | encode | **3.78 GiB/s** | 0.36 GiB/s | 0.99 GiB/s |
158
+ | | 32 MiB | decode | **4.23 GiB/s** | 0.57 GiB/s | 1.67 GiB/s |
159
+
160
+ ### Reusable Python Buffers
161
+
162
+ | Alphabet | Input | Operation | hashcodecs |
163
+ | --- | --- | --- | ---: |
164
+ | Standard | 4 KiB | encode | **15.48 GiB/s** |
165
+ | | 4 KiB | decode | **17.20 GiB/s** |
166
+ | | 1 MiB | encode | **19.18 GiB/s** |
167
+ | | 1 MiB | decode | **23.82 GiB/s** |
168
+ | | 32 MiB | encode | **10.95 GiB/s** |
169
+ | | 32 MiB | decode | **10.47 GiB/s** |
170
+ | URL-safe | 4 KiB | encode | **15.36 GiB/s** |
171
+ | | 4 KiB | decode | **13.48 GiB/s** |
172
+ | | 1 MiB | encode | **19.20 GiB/s** |
173
+ | | 1 MiB | decode | **18.25 GiB/s** |
174
+ | | 32 MiB | encode | **10.87 GiB/s** |
175
+ | | 32 MiB | decode | **9.88 GiB/s** |
176
+
177
+ ## MurmurHash3: Python
178
+
179
+ | Variant | API | Input | hashcodecs | `mmh3` |
180
+ | --- | --- | --- | ---: | ---: |
181
+ | x86 32-bit | one-shot | 4 KiB | **3.80 GiB/s** | 3.72 GiB/s |
182
+ | | | 1 MiB | **4.00 GiB/s** | 3.84 GiB/s |
183
+ | | | 32 MiB | **3.99 GiB/s** | 3.68 GiB/s |
184
+ | | incremental | 4 KiB | **3.60 GiB/s** | 3.59 GiB/s |
185
+ | | | 1 MiB | **4.00 GiB/s** | 3.83 GiB/s |
186
+ | | | 32 MiB | **3.99 GiB/s** | 3.79 GiB/s |
187
+ | x86 128-bit | one-shot | 4 KiB | 8.19 GiB/s | **8.22 GiB/s** |
188
+ | | | 1 MiB | **9.28 GiB/s** | 8.89 GiB/s |
189
+ | | | 32 MiB | **9.18 GiB/s** | 6.25 GiB/s |
190
+ | | incremental | 4 KiB | **7.05 GiB/s** | 0.79 GiB/s |
191
+ | | | 1 MiB | **9.17 GiB/s** | 0.78 GiB/s |
192
+ | | | 32 MiB | **9.12 GiB/s** | 0.79 GiB/s |
193
+ | x64 128-bit | one-shot | 4 KiB | 8.86 GiB/s | **9.48 GiB/s** |
194
+ | | | 1 MiB | 10.07 GiB/s | **10.25 GiB/s** |
195
+ | | | 32 MiB | **9.55 GiB/s** | 6.82 GiB/s |
196
+ | | incremental | 4 KiB | 7.57 GiB/s | **8.02 GiB/s** |
197
+ | | | 1 MiB | **10.07 GiB/s** | 9.18 GiB/s |
198
+ | | | 32 MiB | **9.57 GiB/s** | 7.44 GiB/s |
199
+
200
+ ## SIMD References
201
+
202
+ The SIMD implementation follows the approach described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605), with AVX-512 VBMI and AArch64 NEON backends selected automatically when available.
hashcodecs-0.3.0/README.md ADDED
@@ -0,0 +1,188 @@
1
+ # hashcodecs
2
+
3
+ [![CI](https://img.shields.io/github/actions/workflow/status/kozistr/hashcodecs-rs/ci.yml?branch=main&style=for-the-badge&logo=github)](https://github.com/kozistr/hashcodecs-rs/actions/workflows/ci.yml)
4
+ [![PyPI](https://img.shields.io/pypi/v/hashcodecs?style=for-the-badge&logo=pypi)](https://pypi.org/project/hashcodecs/)
5
+ [![Python](https://img.shields.io/pypi/pyversions/hashcodecs?style=for-the-badge&logo=python)](https://pypi.org/project/hashcodecs/)
6
+ [![License](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-brightgreen?style=for-the-badge)](https://github.com/kozistr/hashcodecs-rs#license)
7
+ [![Downloads](https://img.shields.io/pypi/dm/hashcodecs?style=for-the-badge&label=downloads)](https://pypi.org/project/hashcodecs/)
8
+
9
+ `hashcodecs` provides runtime-dispatched SIMD Base64 codecs and fast, reference-compatible MurmurHash3 functions for Rust and Python.
10
+
11
+ ## Design
12
+
13
+ - Base64 is implemented in this crate with runtime AVX-512 VBMI, AVX2, SSE4.1, SSSE3, and AArch64 NEON dispatch. Portable wheels select the best available SIMD backend and fall back to scalar code otherwise.
14
+ - Rust callers can reuse their own output buffers through the `*_into` APIs. Every backend writes exactly the documented prefix; it never depends on spare capacity after the destination slice.
15
+ - Rust-owned results use the `mimalloc` global allocator.
16
+ - Every MurmurHash3 variant uses runtime-dispatched AVX2 or SSE4.1 pre-mixing where its measured crossover beats the scalar loop. The x64 128-bit AVX2 kernel also selects BMI2 rotations when available. Ordered state transitions remain reference-compatible, and every variant has a portable scalar fallback.
17
+ - The Python Base64 surface follows the familiar `base64` names. Use `import hashcodecs.base64 as base64` when replacing standard Base64 calls.
18
+ - Python callers can reuse a `bytearray` through the `*_into` APIs when returned `bytes` ownership is unnecessary.
19
+ - Large Python calls release the GIL. Strict decoding and valid default-mode decoding write directly into the returned `bytes`; malformed default-mode input uses a CPython-compatible lenient fallback.
20
+
21
+ ## Install
22
+
23
+ ```
24
+ pip3 install hashcodecs
25
+ ```
26
+
27
+ ## Usage
28
+
29
+ ### Rust
30
+
31
+ ```rust
32
+ let encoded = hashcodecs::b64encode(b"hello");
33
+ assert_eq!(encoded, "aGVsbG8=");
34
+
35
+ let mut output = [0_u8; 8];
36
+ let written = hashcodecs::b64encode_into(b"hello", &mut output).unwrap();
37
+ assert_eq!(&output[..written], b"aGVsbG8=");
38
+ assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
39
+ ```
40
+
41
+ ### Python
42
+
43
+ ```python
44
+ import hashcodecs.base64 as base64
45
+ from hashcodecs import murmur3_32, murmur3_x64_128
46
+
47
+ assert base64.b64encode(b'hello') == b'aGVsbG8='
48
+ assert base64.b64decode(b'aGVsbG8=') == b'hello'
49
+ assert murmur3_32(b'hello') == 0x248BFA47
50
+
51
+ output = bytearray(8)
52
+ written = base64.b64encode_into(b'hello', output)
53
+ assert output[:written] == b'aGVsbG8='
54
+
55
+ hasher = murmur3_x64_128(seed=42)
56
+ hasher.update(b'hello')
57
+ assert hasher.hexdigest() == hasher.digest().hex()
58
+ ```
59
+
60
+ Build an installable wheel and source distribution with:
61
+
62
+ ```sh
63
+ uv build
64
+ ```
65
+
66
+ # Benchmark
67
+
68
+ Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
69
+
70
+ Conditions: clean builds, one pinned logical CPU, single-threaded execution, 50 Rust samples, and nine Python samples. Returned-output allocation is included except in the reusable-buffer table. Higher is better.
71
+
72
+ ## Run Locally
73
+
74
+ Comparison crates are development-only dependencies and are not included in consumer builds.
75
+
76
+ ```sh
77
+ cargo bench --bench base64
78
+ cargo bench --bench murmur3
79
+ uv sync --group benchmark --no-install-project
80
+ uv run --no-project python benchmarks/python_base64.py
81
+ uv run --no-project python benchmarks/python_base64.py --into
82
+ uv run --no-project python benchmarks/python_murmur3.py
83
+ uv run --no-project python benchmarks/python_murmur3.py --incremental
84
+ ```
85
+
86
+ To time only this implementation:
87
+
88
+ ```sh
89
+ cargo bench --bench base64 -- hashcodecs
90
+ cargo bench --bench murmur3 -- hashcodecs
91
+ uv run --no-project python benchmarks/python_base64.py --hashcodecs-only
92
+ uv run --no-project python benchmarks/python_murmur3.py --hashcodecs-only
93
+ uv run --no-project python benchmarks/python_murmur3.py --incremental --hashcodecs-only
94
+ ```
95
+
96
+ ## Base64: Rust
97
+
98
+ | Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
99
+ | --- | --- | --- | ---: | ---: | ---: |
100
+ | Standard | 4 KiB | encode | **18.11 GiB/s** | 5.39 GiB/s | 16.94 GiB/s |
101
+ | | 4 KiB | decode | **24.70 GiB/s** | 4.06 GiB/s | 15.73 GiB/s |
102
+ | | 1 MiB | encode | **19.20 GiB/s** | 4.81 GiB/s | 18.19 GiB/s |
103
+ | | 1 MiB | decode | **28.72 GiB/s** | 3.80 GiB/s | 16.66 GiB/s |
104
+ | | 32 MiB | encode | **10.32 GiB/s** | 2.95 GiB/s | 9.97 GiB/s |
105
+ | | 32 MiB | decode | **10.56 GiB/s** | 3.04 GiB/s | 9.70 GiB/s |
106
+ | URL-safe | 4 KiB | encode | **17.95 GiB/s** | 5.40 GiB/s | 16.90 GiB/s |
107
+ | | 4 KiB | decode | **23.39 GiB/s** | 4.05 GiB/s | 15.75 GiB/s |
108
+ | | 1 MiB | encode | **19.19 GiB/s** | 4.80 GiB/s | 18.17 GiB/s |
109
+ | | 1 MiB | decode | **27.19 GiB/s** | 3.79 GiB/s | 16.76 GiB/s |
110
+ | | 32 MiB | encode | **10.33 GiB/s** | 2.98 GiB/s | 10.16 GiB/s |
111
+ | | 32 MiB | decode | **10.80 GiB/s** | 3.10 GiB/s | 9.36 GiB/s |
112
+
113
+ ## MurmurHash3: Rust
114
+
115
+ | Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
116
+ | --- | --- | ---: | ---: | ---: | ---: | ---: |
117
+ | x86 32-bit | 4 KiB | **4.03 GiB/s** | 2.44 GiB/s | 3.82 GiB/s | n/a | 4.02 GiB/s |
118
+ | | 1 MiB | **3.99 GiB/s** | 2.43 GiB/s | 3.79 GiB/s | n/a | 3.97 GiB/s |
119
+ | | 32 MiB | **3.98 GiB/s** | 2.42 GiB/s | 3.65 GiB/s | n/a | 3.81 GiB/s |
120
+ | x86 128-bit | 4 KiB | **8.90 GiB/s** | 4.66 GiB/s | 8.07 GiB/s | n/a | n/a |
121
+ | | 1 MiB | **9.21 GiB/s** | 4.71 GiB/s | 8.22 GiB/s | n/a | n/a |
122
+ | | 32 MiB | **9.12 GiB/s** | 4.55 GiB/s | 5.73 GiB/s | n/a | n/a |
123
+ | x64 128-bit | 4 KiB | **10.08 GiB/s** | 6.38 GiB/s | 8.85 GiB/s | 9.41 GiB/s | 8.91 GiB/s |
124
+ | | 1 MiB | **10.11 GiB/s** | 6.44 GiB/s | 8.87 GiB/s | 9.37 GiB/s | 8.88 GiB/s |
125
+ | | 32 MiB | **9.54 GiB/s** | 5.83 GiB/s | 6.39 GiB/s | 6.99 GiB/s | 6.41 GiB/s |
126
+
127
+ ## Base64: Python
128
+
129
+ Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly into Rust without an input copy.
130
+
131
+ | Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
132
+ | --- | --- | --- | ---: | ---: | ---: |
133
+ | Standard | 4 KiB | encode | 13.73 GiB/s | 0.49 GiB/s | **14.28 GiB/s** |
134
+ | | 4 KiB | decode | **15.64 GiB/s** | 1.15 GiB/s | 8.44 GiB/s |
135
+ | | 1 MiB | encode | **3.68 GiB/s** | 0.47 GiB/s | 3.32 GiB/s |
136
+ | | 1 MiB | decode | 3.64 GiB/s | 0.89 GiB/s | **4.13 GiB/s** |
137
+ | | 32 MiB | encode | **3.75 GiB/s** | 0.47 GiB/s | 3.51 GiB/s |
138
+ | | 32 MiB | decode | 3.81 GiB/s | 0.90 GiB/s | **4.42 GiB/s** |
139
+ | URL-safe | 4 KiB | encode | **13.68 GiB/s** | 0.41 GiB/s | 1.19 GiB/s |
140
+ | | 4 KiB | decode | **12.66 GiB/s** | 0.74 GiB/s | 1.56 GiB/s |
141
+ | | 1 MiB | encode | **3.63 GiB/s** | 0.36 GiB/s | 0.97 GiB/s |
142
+ | | 1 MiB | decode | **4.06 GiB/s** | 0.57 GiB/s | 1.55 GiB/s |
143
+ | | 32 MiB | encode | **3.78 GiB/s** | 0.36 GiB/s | 0.99 GiB/s |
144
+ | | 32 MiB | decode | **4.23 GiB/s** | 0.57 GiB/s | 1.67 GiB/s |
145
+
146
+ ### Reusable Python Buffers
147
+
148
+ | Alphabet | Input | Operation | hashcodecs |
149
+ | --- | --- | --- | ---: |
150
+ | Standard | 4 KiB | encode | **15.48 GiB/s** |
151
+ | | 4 KiB | decode | **17.20 GiB/s** |
152
+ | | 1 MiB | encode | **19.18 GiB/s** |
153
+ | | 1 MiB | decode | **23.82 GiB/s** |
154
+ | | 32 MiB | encode | **10.95 GiB/s** |
155
+ | | 32 MiB | decode | **10.47 GiB/s** |
156
+ | URL-safe | 4 KiB | encode | **15.36 GiB/s** |
157
+ | | 4 KiB | decode | **13.48 GiB/s** |
158
+ | | 1 MiB | encode | **19.20 GiB/s** |
159
+ | | 1 MiB | decode | **18.25 GiB/s** |
160
+ | | 32 MiB | encode | **10.87 GiB/s** |
161
+ | | 32 MiB | decode | **9.88 GiB/s** |
162
+
163
+ ## MurmurHash3: Python
164
+
165
+ | Variant | API | Input | hashcodecs | `mmh3` |
166
+ | --- | --- | --- | ---: | ---: |
167
+ | x86 32-bit | one-shot | 4 KiB | **3.80 GiB/s** | 3.72 GiB/s |
168
+ | | | 1 MiB | **4.00 GiB/s** | 3.84 GiB/s |
169
+ | | | 32 MiB | **3.99 GiB/s** | 3.68 GiB/s |
170
+ | | incremental | 4 KiB | **3.60 GiB/s** | 3.59 GiB/s |
171
+ | | | 1 MiB | **4.00 GiB/s** | 3.83 GiB/s |
172
+ | | | 32 MiB | **3.99 GiB/s** | 3.79 GiB/s |
173
+ | x86 128-bit | one-shot | 4 KiB | 8.19 GiB/s | **8.22 GiB/s** |
174
+ | | | 1 MiB | **9.28 GiB/s** | 8.89 GiB/s |
175
+ | | | 32 MiB | **9.18 GiB/s** | 6.25 GiB/s |
176
+ | | incremental | 4 KiB | **7.05 GiB/s** | 0.79 GiB/s |
177
+ | | | 1 MiB | **9.17 GiB/s** | 0.78 GiB/s |
178
+ | | | 32 MiB | **9.12 GiB/s** | 0.79 GiB/s |
179
+ | x64 128-bit | one-shot | 4 KiB | 8.86 GiB/s | **9.48 GiB/s** |
180
+ | | | 1 MiB | 10.07 GiB/s | **10.25 GiB/s** |
181
+ | | | 32 MiB | **9.55 GiB/s** | 6.82 GiB/s |
182
+ | | incremental | 4 KiB | 7.57 GiB/s | **8.02 GiB/s** |
183
+ | | | 1 MiB | **10.07 GiB/s** | 9.18 GiB/s |
184
+ | | | 32 MiB | **9.57 GiB/s** | 7.44 GiB/s |
185
+
186
+ ## SIMD References
187
+
188
+ The SIMD implementation follows the approach described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605), with AVX-512 VBMI and AArch64 NEON backends selected automatically when available.
{hashcodecs-0.1.0 → hashcodecs-0.3.0}/benches/base64.rs RENAMED
@@ -6,8 +6,8 @@ use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_m
6
6
 
7
7
  mod support;
8
8
 
9
- const LARGE_SIZE: usize = 512 * 1024 * 1024;
10
- const SIZES: [usize; 3] = [4 * 1024, 1024 * 1024, LARGE_SIZE];
9
+ const SIZES: [usize; 3] = [4 * 1024, 1024 * 1024, 32 * 1024 * 1024];
10
+ const SAMPLE_SIZE: usize = 50;
11
11
 
12
12
  fn data(size: usize) -> Vec<u8> {
13
13
  (0..size)
@@ -23,10 +23,6 @@ macro_rules! benchmark {
23
23
  };
24
24
  }
25
25
 
26
- fn sample_size(size: usize) -> usize {
27
- if size == LARGE_SIZE { 10 } else { 30 }
28
- }
29
-
30
26
  fn base64(c: &mut Criterion) {
31
27
  support::pin_to_one_cpu();
32
28
  standard_encode(c);
@@ -46,7 +42,7 @@ fn standard_encode(c: &mut Criterion) {
46
42
  );
47
43
  assert_eq!(base64_turbo::STANDARD.encode(&input), expected);
48
44
 
49
- group.sample_size(sample_size(size));
45
+ group.sample_size(SAMPLE_SIZE);
50
46
  group.throughput(Throughput::Bytes(size as u64));
51
47
  benchmark!(group, size, &input, "hashcodecs", hashcodecs::b64encode);
52
48
  benchmark!(group, size, &input, "base64", |input: &[u8]| {
@@ -70,7 +66,7 @@ fn urlsafe_encode(c: &mut Criterion) {
70
66
  );
71
67
  assert_eq!(base64_turbo::URL_SAFE.encode(&input), expected);
72
68
 
73
- group.sample_size(sample_size(size));
69
+ group.sample_size(SAMPLE_SIZE);
74
70
  group.throughput(Throughput::Bytes(size as u64));
75
71
  benchmark!(
76
72
  group,
@@ -102,7 +98,7 @@ fn standard_decode(c: &mut Criterion) {
102
98
  );
103
99
  assert_eq!(base64_turbo::STANDARD.decode(&input).unwrap(), expected);
104
100
 
105
- group.sample_size(sample_size(size));
101
+ group.sample_size(SAMPLE_SIZE);
106
102
  group.throughput(Throughput::Bytes(size as u64));
107
103
  benchmark!(
108
104
  group,
@@ -140,7 +136,7 @@ fn urlsafe_decode(c: &mut Criterion) {
140
136
  );
141
137
  assert_eq!(base64_turbo::URL_SAFE.decode(&input).unwrap(), expected);
142
138
 
143
- group.sample_size(sample_size(size));
139
+ group.sample_size(SAMPLE_SIZE);
144
140
  group.throughput(Throughput::Bytes(size as u64));
145
141
  benchmark!(
146
142
  group,
{hashcodecs-0.1.0 → hashcodecs-0.3.0}/benches/murmur3.rs RENAMED
@@ -6,8 +6,8 @@ use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_m
6
6
 
7
7
  mod support;
8
8
 
9
- const LARGE_SIZE: usize = 512 * 1024 * 1024;
10
- const SIZES: [usize; 3] = [4 * 1024, 1024 * 1024, LARGE_SIZE];
9
+ const SIZES: [usize; 3] = [4 * 1024, 1024 * 1024, 32 * 1024 * 1024];
10
+ const SAMPLE_SIZE: usize = 50;
11
11
 
12
12
  fn data(size: usize) -> Vec<u8> {
13
13
  (0..size)
@@ -52,7 +52,7 @@ fn x86_32(c: &mut Criterion) {
52
52
  );
53
53
  assert_eq!(murmurs::murmur3_x86_32(&input, 42), expected);
54
54
  assert_eq!(mm3h::murmurhash3_32_with_seed(&input, 42), expected);
55
- group.sample_size(if size == LARGE_SIZE { 10 } else { 30 });
55
+ group.sample_size(SAMPLE_SIZE);
56
56
  group.throughput(Throughput::Bytes(size as u64));
57
57
  benchmark!(group, size, &input, "hashcodecs", |input: &[u8]| {
58
58
  hashcodecs::murmur3_x86_32(input, 42)
@@ -80,7 +80,7 @@ fn x86_128(c: &mut Criterion) {
80
80
  x86_128_as_u128(expected)
81
81
  );
82
82
  assert_eq!(murmurs::murmur3_x86_128(&input, 42), expected);
83
- group.sample_size(if size == LARGE_SIZE { 10 } else { 30 });
83
+ group.sample_size(SAMPLE_SIZE);
84
84
  group.throughput(Throughput::Bytes(size as u64));
85
85
  benchmark!(group, size, &input, "hashcodecs", |input: &[u8]| {
86
86
  hashcodecs::murmur3_x86_128(input, 42)
@@ -108,7 +108,7 @@ fn x64_128(c: &mut Criterion) {
108
108
  assert_eq!(murmurs::murmur3_x64_128(&input, 42), expected);
109
109
  assert_eq!(fastmurmur3::murmur3_x64_128(&input, 42), expected_u128);
110
110
  assert_eq!(mm3h::murmurhash3_128_with_seed(&input, 42), expected_u128);
111
- group.sample_size(if size == LARGE_SIZE { 10 } else { 30 });
111
+ group.sample_size(SAMPLE_SIZE);
112
112
  group.throughput(Throughput::Bytes(size as u64));
113
113
  benchmark!(group, size, &input, "hashcodecs", |input: &[u8]| {
114
114
  hashcodecs::murmur3_x64_128(input, 42)
{hashcodecs-0.1.0 → hashcodecs-0.3.0}/build.rs RENAMED
@@ -1,4 +1,6 @@
1
1
  fn main() {
2
+ println!("cargo::rustc-check-cfg=cfg(coverage)");
3
+
2
4
  let is_macos = std::env::var("CARGO_CFG_TARGET_OS").is_ok_and(|target| target == "macos");
3
5
  let is_extension = std::env::var_os("CARGO_FEATURE_EXTENSION_MODULE").is_some();
4
6
 
{hashcodecs-0.1.0 → hashcodecs-0.3.0}/hatch_build.py RENAMED
@@ -15,42 +15,42 @@ class CustomBuildHook(BuildHookInterface[Any]):
15
15
  """Build the portable ABI3 extension and give the wheel its native tag."""
16
16
 
17
17
  def initialize(self, version: str, build_data: dict[str, Any]) -> None:
18
- if self.target_name != "wheel":
18
+ if self.target_name != 'wheel':
19
19
  return
20
20
 
21
21
  root = Path(self.root)
22
22
  subprocess.run(
23
23
  [
24
- "cargo",
25
- "build",
26
- "--manifest-path",
27
- str(root / "Cargo.toml"),
28
- "--release",
29
- "--features",
30
- "extension-module",
24
+ 'cargo',
25
+ 'build',
26
+ '--manifest-path',
27
+ str(root / 'Cargo.toml'),
28
+ '--release',
29
+ '--features',
30
+ 'extension-module',
31
31
  ],
32
32
  check=True,
33
33
  cwd=root,
34
34
  )
35
35
 
36
- extension = root / "target" / "release" / self._library_name()
37
- build_data["force_include"] = {str(extension): f"hashcodecs/_hashcodecs{self._extension_suffix()}"}
38
- build_data["pure_python"] = False
39
- build_data["tag"] = self._wheel_tag()
36
+ extension = root / 'target' / 'release' / self._library_name()
37
+ build_data['force_include'] = {str(extension): f'hashcodecs/_hashcodecs{self._extension_suffix()}'}
38
+ build_data['pure_python'] = False
39
+ build_data['tag'] = self._wheel_tag()
40
40
 
41
41
  @staticmethod
42
42
  def _library_name() -> str:
43
43
  system = platform.system()
44
- if system == "Windows":
45
- return "hashcodecs.dll"
46
- if system == "Darwin":
47
- return "libhashcodecs.dylib"
48
- return "libhashcodecs.so"
44
+ if system == 'Windows':
45
+ return 'hashcodecs.dll'
46
+ if system == 'Darwin':
47
+ return 'libhashcodecs.dylib'
48
+ return 'libhashcodecs.so'
49
49
 
50
50
  @staticmethod
51
51
  def _extension_suffix() -> str:
52
- return ".pyd" if platform.system() == "Windows" else ".so"
52
+ return '.pyd' if platform.system() == 'Windows' else '.so'
53
53
 
54
54
  @staticmethod
55
55
  def _wheel_tag() -> str:
56
- return f"cp310-abi3-{next(sys_tags()).platform}"
56
+ return f'cp310-abi3-{next(sys_tags()).platform}'
{hashcodecs-0.1.0 → hashcodecs-0.3.0}/pyproject.toml RENAMED
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "hashcodecs"
7
- version = "0.1.0"
7
+ version = "0.3.0"
8
8
  description = "SIMD-accelerated Base64 and MurmurHash3 codecs"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -21,9 +21,12 @@ classifiers = [
21
21
  dev = [
22
22
  "pytest>=9,<10",
23
23
  "pytest-cov>=7,<8",
24
- "pybase64>=1.4.3,<2",
25
24
  "ruff>=0.16,<0.17",
26
25
  ]
26
+ benchmark = [
27
+ "mmh3>=5.2.1,<6",
28
+ "pybase64>=1.4.3,<2",
29
+ ]
27
30
 
28
31
  [tool.hatch.build.targets.wheel]
29
32
  packages = ["python/hashcodecs"]
@@ -37,11 +40,10 @@ include = [
37
40
  "/LICENSE*",
38
41
  "/README.md",
39
42
  "/benches",
40
- "/benchmarks",
41
43
  "/python",
42
44
  "/src",
43
- "/tests",
44
45
  ]
46
+ exclude = ["/src/base64/tests.rs"]
45
47
 
46
48
  [tool.hatch.build.hooks.custom]
47
49
  path = "hatch_build.py"
@@ -50,7 +52,7 @@ path = "hatch_build.py"
50
52
  build = "cp310-*"
51
53
  build-frontend = "uv"
52
54
  skip = "*-musllinux*"
53
- test-command = "python -c \"import hashcodecs; assert hashcodecs.b64decode(hashcodecs.b64encode(b'wheel')) == b'wheel'\""
55
+ test-command = "python -c \"import hashcodecs; assert hashcodecs.b64decode(hashcodecs.b64encode(b'hashcodecs')) == b'hashcodecs'\""
54
56
 
55
57
  [tool.cibuildwheel.linux]
56
58
  before-all = "curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal"
@@ -64,6 +66,7 @@ line-length = 119
64
66
  target-version = "py312"
65
67
 
66
68
  [tool.ruff.lint]
69
+ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
67
70
  select = [
68
71
  "E",
69
72
  "F",
@@ -77,11 +80,24 @@ select = [
77
80
  "PTH",
78
81
  "PT",
79
82
  "PYI",
83
+ "Q",
80
84
  "RET",
81
85
  "RUF",
82
86
  "SIM",
83
87
  ]
84
88
 
89
+ [tool.ruff.lint.flake8-quotes]
90
+ docstring-quotes = "double"
91
+ inline-quotes = "single"
92
+
93
+ [tool.ruff.lint.isort]
94
+ combine-as-imports = false
95
+ detect-same-package = true
96
+ force-sort-within-sections = false
97
+
98
+ [tool.ruff.format]
99
+ quote-style = "single"
100
+
85
101
  [tool.coverage.run]
86
102
  branch = true
87
103
  source = ["hashcodecs"]