hashcodecs 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/Cargo.lock +1 -1
  2. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/Cargo.toml +1 -1
  3. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/PKG-INFO +80 -57
  4. hashcodecs-0.3.0/README.md +188 -0
  5. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/hatch_build.py +19 -19
  6. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/pyproject.toml +15 -1
  7. hashcodecs-0.3.0/python/hashcodecs/__init__.py +45 -0
  8. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/python/hashcodecs/_hashcodecs.pyi +7 -0
  9. hashcodecs-0.3.0/python/hashcodecs/base64.py +59 -0
  10. hashcodecs-0.3.0/python/hashcodecs/base64.pyi +23 -0
  11. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/python/hashcodecs/murmur3.py +6 -6
  12. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/base64/aarch64.rs +98 -42
  13. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/base64/x86.rs +65 -57
  14. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/base64/x86_avx512.rs +58 -10
  15. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/python/base64.rs +122 -4
  16. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/python.rs +3 -1
  17. hashcodecs-0.2.0/README.md +0 -165
  18. hashcodecs-0.2.0/python/hashcodecs/__init__.py +0 -26
  19. hashcodecs-0.2.0/python/hashcodecs/base64.py +0 -33
  20. hashcodecs-0.2.0/python/hashcodecs/base64.pyi +0 -12
  21. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/.gitignore +0 -0
  22. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/LICENSE +0 -0
  23. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/LICENSE-MIT +0 -0
  24. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/benches/base64.rs +0 -0
  25. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/benches/murmur3.rs +0 -0
  26. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/benches/support/mod.rs +0 -0
  27. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/build.rs +0 -0
  28. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/python/hashcodecs/py.typed +0 -0
  29. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/base64/dispatch.rs +0 -0
  30. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/base64.rs +0 -0
  31. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/lib.rs +0 -0
  32. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/murmur3/dispatch.rs +0 -0
  33. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/murmur3/x86.rs +0 -0
  34. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/murmur3.rs +0 -0
  35. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/python/buffer.rs +0 -0
  36. {hashcodecs-0.2.0 → hashcodecs-0.3.0}/src/python/murmur3.rs +0 -0
{hashcodecs-0.2.0 → hashcodecs-0.3.0}/Cargo.lock RENAMED
@@ -194,7 +194,7 @@ dependencies = [
194
194
 
195
195
  [[package]]
196
196
  name = "hashcodecs"
197
- version = "0.2.0"
197
+ version = "0.3.0"
198
198
  dependencies = [
199
199
  "base64",
200
200
  "base64-turbo",
{hashcodecs-0.2.0 → hashcodecs-0.3.0}/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "hashcodecs"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  edition = "2024"
5
5
  rust-version = "1.89"
6
6
  description = "SIMD-accelerated Base64 codecs and fast MurmurHash3 implementations"
{hashcodecs-0.2.0 → hashcodecs-0.3.0}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hashcodecs
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: SIMD-accelerated Base64 and MurmurHash3 codecs
5
5
  License-Expression: MIT OR Apache-2.0
6
6
  License-File: LICENSE
@@ -29,6 +29,7 @@ Description-Content-Type: text/markdown
29
29
  - Rust-owned results use the `mimalloc` global allocator.
30
30
  - Every MurmurHash3 variant uses runtime-dispatched AVX2 or SSE4.1 pre-mixing where its measured crossover beats the scalar loop. The x64 128-bit AVX2 kernel also selects BMI2 rotations when available. Ordered state transitions remain reference-compatible, and every variant has a portable scalar fallback.
31
31
  - The Python Base64 surface follows the familiar `base64` names. Use `import hashcodecs.base64 as base64` when replacing standard Base64 calls.
32
+ - Python callers can reuse a `bytearray` through the `*_into` APIs when returned `bytes` ownership is unnecessary.
32
33
  - Large Python calls release the GIL. Strict decoding and valid default-mode decoding write directly into the returned `bytes`; malformed default-mode input uses a CPython-compatible lenient fallback.
33
34
 
34
35
  ## Install
@@ -57,12 +58,16 @@ assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
57
58
  import hashcodecs.base64 as base64
58
59
  from hashcodecs import murmur3_32, murmur3_x64_128
59
60
 
60
- assert base64.b64encode(b"hello") == b"aGVsbG8="
61
- assert base64.b64decode(b"aGVsbG8=") == b"hello"
62
- assert murmur3_32(b"hello") == 0x248BFA47
61
+ assert base64.b64encode(b'hello') == b'aGVsbG8='
62
+ assert base64.b64decode(b'aGVsbG8=') == b'hello'
63
+ assert murmur3_32(b'hello') == 0x248BFA47
64
+
65
+ output = bytearray(8)
66
+ written = base64.b64encode_into(b'hello', output)
67
+ assert output[:written] == b'aGVsbG8='
63
68
 
64
69
  hasher = murmur3_x64_128(seed=42)
65
- hasher.update(b"hello")
70
+ hasher.update(b'hello')
66
71
  assert hasher.hexdigest() == hasher.digest().hex()
67
72
  ```
68
73
 
@@ -76,7 +81,7 @@ uv build
76
81
 
77
82
  Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
78
83
 
79
- Conditions: clean builds, one pinned logical CPU, single-threaded execution, and output allocation included. Higher is better.
84
+ Conditions: clean builds, one pinned logical CPU, single-threaded execution, 50 Rust samples, and nine Python samples. Returned-output allocation is included except in the reusable-buffer table. Higher is better.
80
85
 
81
86
  ## Run Locally
82
87
 
@@ -87,6 +92,7 @@ cargo bench --bench base64
87
92
  cargo bench --bench murmur3
88
93
  uv sync --group benchmark --no-install-project
89
94
  uv run --no-project python benchmarks/python_base64.py
95
+ uv run --no-project python benchmarks/python_base64.py --into
90
96
  uv run --no-project python benchmarks/python_murmur3.py
91
97
  uv run --no-project python benchmarks/python_murmur3.py --incremental
92
98
  ```
@@ -105,32 +111,32 @@ uv run --no-project python benchmarks/python_murmur3.py --incremental --hashcode
105
111
 
106
112
  | Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
107
113
  | --- | --- | --- | ---: | ---: | ---: |
108
- | Standard | 4 KiB | encode | **18.17 GiB/s** | 5.40 GiB/s | 16.97 GiB/s |
109
- | | 4 KiB | decode | **16.26 GiB/s** | 4.05 GiB/s | 15.76 GiB/s |
110
- | | 1 MiB | encode | **19.29 GiB/s** | 4.79 GiB/s | 18.28 GiB/s |
111
- | | 1 MiB | decode | **18.41 GiB/s** | 3.79 GiB/s | 16.93 GiB/s |
112
- | | 32 MiB | encode | **10.29 GiB/s** | 3.08 GiB/s | 10.24 GiB/s |
113
- | | 32 MiB | decode | **10.12 GiB/s** | 3.12 GiB/s | 9.83 GiB/s |
114
- | URL-safe | 4 KiB | encode | **18.19 GiB/s** | 5.39 GiB/s | 16.98 GiB/s |
115
- | | 4 KiB | decode | 14.28 GiB/s | 4.02 GiB/s | **15.77 GiB/s** |
116
- | | 1 MiB | encode | **19.27 GiB/s** | 4.82 GiB/s | 18.32 GiB/s |
117
- | | 1 MiB | decode | 16.28 GiB/s | 3.83 GiB/s | **16.99 GiB/s** |
118
- | | 32 MiB | encode | 10.35 GiB/s | 3.11 GiB/s | **10.45 GiB/s** |
119
- | | 32 MiB | decode | 9.83 GiB/s | 3.17 GiB/s | **9.85 GiB/s** |
114
+ | Standard | 4 KiB | encode | **18.11 GiB/s** | 5.39 GiB/s | 16.94 GiB/s |
115
+ | | 4 KiB | decode | **24.70 GiB/s** | 4.06 GiB/s | 15.73 GiB/s |
116
+ | | 1 MiB | encode | **19.20 GiB/s** | 4.81 GiB/s | 18.19 GiB/s |
117
+ | | 1 MiB | decode | **28.72 GiB/s** | 3.80 GiB/s | 16.66 GiB/s |
118
+ | | 32 MiB | encode | **10.32 GiB/s** | 2.95 GiB/s | 9.97 GiB/s |
119
+ | | 32 MiB | decode | **10.56 GiB/s** | 3.04 GiB/s | 9.70 GiB/s |
120
+ | URL-safe | 4 KiB | encode | **17.95 GiB/s** | 5.40 GiB/s | 16.90 GiB/s |
121
+ | | 4 KiB | decode | **23.39 GiB/s** | 4.05 GiB/s | 15.75 GiB/s |
122
+ | | 1 MiB | encode | **19.19 GiB/s** | 4.80 GiB/s | 18.17 GiB/s |
123
+ | | 1 MiB | decode | **27.19 GiB/s** | 3.79 GiB/s | 16.76 GiB/s |
124
+ | | 32 MiB | encode | **10.33 GiB/s** | 2.98 GiB/s | 10.16 GiB/s |
125
+ | | 32 MiB | decode | **10.80 GiB/s** | 3.10 GiB/s | 9.36 GiB/s |
120
126
 
121
127
  ## MurmurHash3: Rust
122
128
 
123
129
  | Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
124
130
  | --- | --- | ---: | ---: | ---: | ---: | ---: |
125
- | x86 32-bit | 4 KiB | 4.01 GiB/s | 2.42 GiB/s | 3.83 GiB/s | n/a | **4.03 GiB/s** |
126
- | | 1 MiB | **4.00 GiB/s** | 2.44 GiB/s | 3.79 GiB/s | n/a | 3.98 GiB/s |
127
- | | 32 MiB | **3.99 GiB/s** | 2.42 GiB/s | 3.67 GiB/s | n/a | 3.83 GiB/s |
128
- | x86 128-bit | 4 KiB | **8.93 GiB/s** | 4.66 GiB/s | 8.09 GiB/s | n/a | n/a |
129
- | | 1 MiB | **9.24 GiB/s** | 4.72 GiB/s | 8.22 GiB/s | n/a | n/a |
130
- | | 32 MiB | **9.13 GiB/s** | 4.56 GiB/s | 5.92 GiB/s | n/a | n/a |
131
- | x64 128-bit | 4 KiB | **10.08 GiB/s** | 6.38 GiB/s | 8.75 GiB/s | 9.41 GiB/s | 8.93 GiB/s |
132
- | | 1 MiB | **10.13 GiB/s** | 6.45 GiB/s | 8.90 GiB/s | 9.41 GiB/s | 8.90 GiB/s |
133
- | | 32 MiB | **9.57 GiB/s** | 5.89 GiB/s | 6.42 GiB/s | 6.97 GiB/s | 6.44 GiB/s |
131
+ | x86 32-bit | 4 KiB | **4.03 GiB/s** | 2.44 GiB/s | 3.82 GiB/s | n/a | 4.02 GiB/s |
132
+ | | 1 MiB | **3.99 GiB/s** | 2.43 GiB/s | 3.79 GiB/s | n/a | 3.97 GiB/s |
133
+ | | 32 MiB | **3.98 GiB/s** | 2.42 GiB/s | 3.65 GiB/s | n/a | 3.81 GiB/s |
134
+ | x86 128-bit | 4 KiB | **8.90 GiB/s** | 4.66 GiB/s | 8.07 GiB/s | n/a | n/a |
135
+ | | 1 MiB | **9.21 GiB/s** | 4.71 GiB/s | 8.22 GiB/s | n/a | n/a |
136
+ | | 32 MiB | **9.12 GiB/s** | 4.55 GiB/s | 5.73 GiB/s | n/a | n/a |
137
+ | x64 128-bit | 4 KiB | **10.08 GiB/s** | 6.38 GiB/s | 8.85 GiB/s | 9.41 GiB/s | 8.91 GiB/s |
138
+ | | 1 MiB | **10.11 GiB/s** | 6.44 GiB/s | 8.87 GiB/s | 9.37 GiB/s | 8.88 GiB/s |
139
+ | | 32 MiB | **9.54 GiB/s** | 5.83 GiB/s | 6.39 GiB/s | 6.99 GiB/s | 6.41 GiB/s |
134
140
 
135
141
  ## Base64: Python
136
142
 
@@ -138,42 +144,59 @@ Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly i
138
144
 
139
145
  | Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
140
146
  | --- | --- | --- | ---: | ---: | ---: |
141
- | Standard | 4 KiB | encode | 13.74 GiB/s | 0.49 GiB/s | **14.05 GiB/s** |
142
- | | 4 KiB | decode | **11.95 GiB/s** | 1.15 GiB/s | 8.47 GiB/s |
143
- | | 1 MiB | encode | **3.70 GiB/s** | 0.47 GiB/s | 3.40 GiB/s |
144
- | | 1 MiB | decode | **4.29 GiB/s** | 0.89 GiB/s | 4.24 GiB/s |
145
- | | 32 MiB | encode | **3.81 GiB/s** | 0.47 GiB/s | 3.61 GiB/s |
146
- | | 32 MiB | decode | **4.58 GiB/s** | 0.91 GiB/s | **4.58 GiB/s** |
147
- | URL-safe | 4 KiB | encode | **13.54 GiB/s** | 0.41 GiB/s | 1.19 GiB/s |
148
- | | 4 KiB | decode | **8.89 GiB/s** | 0.73 GiB/s | 1.56 GiB/s |
149
- | | 1 MiB | encode | **3.70 GiB/s** | 0.36 GiB/s | 0.97 GiB/s |
150
- | | 1 MiB | decode | **4.07 GiB/s** | 0.57 GiB/s | 1.56 GiB/s |
151
- | | 32 MiB | encode | **3.88 GiB/s** | 0.37 GiB/s | 0.99 GiB/s |
152
- | | 32 MiB | decode | **4.30 GiB/s** | 0.58 GiB/s | 1.67 GiB/s |
147
+ | Standard | 4 KiB | encode | 13.73 GiB/s | 0.49 GiB/s | **14.28 GiB/s** |
148
+ | | 4 KiB | decode | **15.64 GiB/s** | 1.15 GiB/s | 8.44 GiB/s |
149
+ | | 1 MiB | encode | **3.68 GiB/s** | 0.47 GiB/s | 3.32 GiB/s |
150
+ | | 1 MiB | decode | 3.64 GiB/s | 0.89 GiB/s | **4.13 GiB/s** |
151
+ | | 32 MiB | encode | **3.75 GiB/s** | 0.47 GiB/s | 3.51 GiB/s |
152
+ | | 32 MiB | decode | 3.81 GiB/s | 0.90 GiB/s | **4.42 GiB/s** |
153
+ | URL-safe | 4 KiB | encode | **13.68 GiB/s** | 0.41 GiB/s | 1.19 GiB/s |
154
+ | | 4 KiB | decode | **12.66 GiB/s** | 0.74 GiB/s | 1.56 GiB/s |
155
+ | | 1 MiB | encode | **3.63 GiB/s** | 0.36 GiB/s | 0.97 GiB/s |
156
+ | | 1 MiB | decode | **4.06 GiB/s** | 0.57 GiB/s | 1.55 GiB/s |
157
+ | | 32 MiB | encode | **3.78 GiB/s** | 0.36 GiB/s | 0.99 GiB/s |
158
+ | | 32 MiB | decode | **4.23 GiB/s** | 0.57 GiB/s | 1.67 GiB/s |
159
+
160
+ ### Reusable Python Buffers
161
+
162
+ | Alphabet | Input | Operation | hashcodecs |
163
+ | --- | --- | --- | ---: |
164
+ | Standard | 4 KiB | encode | **15.48 GiB/s** |
165
+ | | 4 KiB | decode | **17.20 GiB/s** |
166
+ | | 1 MiB | encode | **19.18 GiB/s** |
167
+ | | 1 MiB | decode | **23.82 GiB/s** |
168
+ | | 32 MiB | encode | **10.95 GiB/s** |
169
+ | | 32 MiB | decode | **10.47 GiB/s** |
170
+ | URL-safe | 4 KiB | encode | **15.36 GiB/s** |
171
+ | | 4 KiB | decode | **13.48 GiB/s** |
172
+ | | 1 MiB | encode | **19.20 GiB/s** |
173
+ | | 1 MiB | decode | **18.25 GiB/s** |
174
+ | | 32 MiB | encode | **10.87 GiB/s** |
175
+ | | 32 MiB | decode | **9.88 GiB/s** |
153
176
 
154
177
  ## MurmurHash3: Python
155
178
 
156
179
  | Variant | API | Input | hashcodecs | `mmh3` |
157
180
  | --- | --- | --- | ---: | ---: |
158
- | x86 32-bit | one-shot | 4 KiB | **3.79 GiB/s** | 3.72 GiB/s |
159
- | | | 1 MiB | **3.99 GiB/s** | 3.84 GiB/s |
160
- | | | 32 MiB | **3.99 GiB/s** | 3.69 GiB/s |
161
- | | incremental | 4 KiB | **3.60 GiB/s** | 3.59 GiB/s |
181
+ | x86 32-bit | one-shot | 4 KiB | **3.80 GiB/s** | 3.72 GiB/s |
162
182
  | | | 1 MiB | **4.00 GiB/s** | 3.84 GiB/s |
163
- | | | 32 MiB | **3.99 GiB/s** | 3.80 GiB/s |
164
- | x86 128-bit | one-shot | 4 KiB | 8.01 GiB/s | **8.15 GiB/s** |
165
- | | | 1 MiB | **9.17 GiB/s** | 8.88 GiB/s |
166
- | | | 32 MiB | **9.08 GiB/s** | 6.29 GiB/s |
167
- | | incremental | 4 KiB | **7.07 GiB/s** | 0.79 GiB/s |
168
- | | | 1 MiB | **9.21 GiB/s** | 0.80 GiB/s |
169
- | | | 32 MiB | **9.17 GiB/s** | 0.79 GiB/s |
170
- | x64 128-bit | one-shot | 4 KiB | 8.78 GiB/s | **9.48 GiB/s** |
171
- | | | 1 MiB | 10.00 GiB/s | **10.26 GiB/s** |
172
- | | | 32 MiB | **9.64 GiB/s** | 6.88 GiB/s |
173
- | | incremental | 4 KiB | 7.75 GiB/s | **8.02 GiB/s** |
174
- | | | 1 MiB | **10.09 GiB/s** | 9.36 GiB/s |
175
- | | | 32 MiB | **9.62 GiB/s** | 7.58 GiB/s |
183
+ | | | 32 MiB | **3.99 GiB/s** | 3.68 GiB/s |
184
+ | | incremental | 4 KiB | **3.60 GiB/s** | 3.59 GiB/s |
185
+ | | | 1 MiB | **4.00 GiB/s** | 3.83 GiB/s |
186
+ | | | 32 MiB | **3.99 GiB/s** | 3.79 GiB/s |
187
+ | x86 128-bit | one-shot | 4 KiB | 8.19 GiB/s | **8.22 GiB/s** |
188
+ | | | 1 MiB | **9.28 GiB/s** | 8.89 GiB/s |
189
+ | | | 32 MiB | **9.18 GiB/s** | 6.25 GiB/s |
190
+ | | incremental | 4 KiB | **7.05 GiB/s** | 0.79 GiB/s |
191
+ | | | 1 MiB | **9.17 GiB/s** | 0.78 GiB/s |
192
+ | | | 32 MiB | **9.12 GiB/s** | 0.79 GiB/s |
193
+ | x64 128-bit | one-shot | 4 KiB | 8.86 GiB/s | **9.48 GiB/s** |
194
+ | | | 1 MiB | 10.07 GiB/s | **10.25 GiB/s** |
195
+ | | | 32 MiB | **9.55 GiB/s** | 6.82 GiB/s |
196
+ | | incremental | 4 KiB | 7.57 GiB/s | **8.02 GiB/s** |
197
+ | | | 1 MiB | **10.07 GiB/s** | 9.18 GiB/s |
198
+ | | | 32 MiB | **9.57 GiB/s** | 7.44 GiB/s |
176
199
 
177
200
  ## SIMD References
178
201
 
179
- The AVX2 block structure uses the 24-byte encode and 32-byte decode arrangement described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605). The AVX-512 VBMI path processes 48-byte encode and 64-byte decode blocks with cross-lane byte permutations, while AArch64 uses 48-byte NEON encoding and 64-byte NEON decoding blocks. The Rust 1.89 MSRV is the first stable release that exposes the required AVX-512 intrinsics.
202
+ The SIMD implementation follows the approach described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605), with AVX-512 VBMI and AArch64 NEON backends selected automatically when available.
hashcodecs-0.3.0/README.md ADDED
@@ -0,0 +1,188 @@
1
+ # hashcodecs
2
+
3
+ [![CI](https://img.shields.io/github/actions/workflow/status/kozistr/hashcodecs-rs/ci.yml?branch=main&style=for-the-badge&logo=github)](https://github.com/kozistr/hashcodecs-rs/actions/workflows/ci.yml)
4
+ [![PyPI](https://img.shields.io/pypi/v/hashcodecs?style=for-the-badge&logo=pypi)](https://pypi.org/project/hashcodecs/)
5
+ [![Python](https://img.shields.io/pypi/pyversions/hashcodecs?style=for-the-badge&logo=python)](https://pypi.org/project/hashcodecs/)
6
+ [![License](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-brightgreen?style=for-the-badge)](https://github.com/kozistr/hashcodecs-rs#license)
7
+ [![Downloads](https://img.shields.io/pypi/dm/hashcodecs?style=for-the-badge&label=downloads)](https://pypi.org/project/hashcodecs/)
8
+
9
+ `hashcodecs` provides runtime-dispatched SIMD Base64 codecs and fast, reference-compatible MurmurHash3 functions for Rust and Python.
10
+
11
+ ## Design
12
+
13
+ - Base64 is implemented in this crate with runtime AVX-512 VBMI, AVX2, SSE4.1, SSSE3, and AArch64 NEON dispatch. Portable wheels select the best available SIMD backend and fall back to scalar code otherwise.
14
+ - Rust callers can reuse their own output buffers through the `*_into` APIs. Every backend writes exactly the documented prefix; it never depends on spare capacity after the destination slice.
15
+ - Rust-owned results use the `mimalloc` global allocator.
16
+ - Every MurmurHash3 variant uses runtime-dispatched AVX2 or SSE4.1 pre-mixing where its measured crossover beats the scalar loop. The x64 128-bit AVX2 kernel also selects BMI2 rotations when available. Ordered state transitions remain reference-compatible, and every variant has a portable scalar fallback.
17
+ - The Python Base64 surface follows the familiar `base64` names. Use `import hashcodecs.base64 as base64` when replacing standard Base64 calls.
18
+ - Python callers can reuse a `bytearray` through the `*_into` APIs when returned `bytes` ownership is unnecessary.
19
+ - Large Python calls release the GIL. Strict decoding and valid default-mode decoding write directly into the returned `bytes`; malformed default-mode input uses a CPython-compatible lenient fallback.
20
+
21
+ ## Install
22
+
23
+ ```
24
+ pip3 install hashcodecs
25
+ ```
26
+
27
+ ## Usage
28
+
29
+ ### Rust
30
+
31
+ ```rust
32
+ let encoded = hashcodecs::b64encode(b"hello");
33
+ assert_eq!(encoded, "aGVsbG8=");
34
+
35
+ let mut output = [0_u8; 8];
36
+ let written = hashcodecs::b64encode_into(b"hello", &mut output).unwrap();
37
+ assert_eq!(&output[..written], b"aGVsbG8=");
38
+ assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
39
+ ```
40
+
41
+ ### Python
42
+
43
+ ```python
44
+ import hashcodecs.base64 as base64
45
+ from hashcodecs import murmur3_32, murmur3_x64_128
46
+
47
+ assert base64.b64encode(b'hello') == b'aGVsbG8='
48
+ assert base64.b64decode(b'aGVsbG8=') == b'hello'
49
+ assert murmur3_32(b'hello') == 0x248BFA47
50
+
51
+ output = bytearray(8)
52
+ written = base64.b64encode_into(b'hello', output)
53
+ assert output[:written] == b'aGVsbG8='
54
+
55
+ hasher = murmur3_x64_128(seed=42)
56
+ hasher.update(b'hello')
57
+ assert hasher.hexdigest() == hasher.digest().hex()
58
+ ```
59
+
60
+ Build an installable wheel and source distribution with:
61
+
62
+ ```sh
63
+ uv build
64
+ ```
65
+
66
+ # Benchmark
67
+
68
+ Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
69
+
70
+ Conditions: clean builds, one pinned logical CPU, single-threaded execution, 50 Rust samples, and nine Python samples. Returned-output allocation is included except in the reusable-buffer table. Higher is better.
71
+
72
+ ## Run Locally
73
+
74
+ Comparison crates are development-only dependencies and are not included in consumer builds.
75
+
76
+ ```sh
77
+ cargo bench --bench base64
78
+ cargo bench --bench murmur3
79
+ uv sync --group benchmark --no-install-project
80
+ uv run --no-project python benchmarks/python_base64.py
81
+ uv run --no-project python benchmarks/python_base64.py --into
82
+ uv run --no-project python benchmarks/python_murmur3.py
83
+ uv run --no-project python benchmarks/python_murmur3.py --incremental
84
+ ```
85
+
86
+ To time only this implementation:
87
+
88
+ ```sh
89
+ cargo bench --bench base64 -- hashcodecs
90
+ cargo bench --bench murmur3 -- hashcodecs
91
+ uv run --no-project python benchmarks/python_base64.py --hashcodecs-only
92
+ uv run --no-project python benchmarks/python_murmur3.py --hashcodecs-only
93
+ uv run --no-project python benchmarks/python_murmur3.py --incremental --hashcodecs-only
94
+ ```
95
+
96
+ ## Base64: Rust
97
+
98
+ | Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
99
+ | --- | --- | --- | ---: | ---: | ---: |
100
+ | Standard | 4 KiB | encode | **18.11 GiB/s** | 5.39 GiB/s | 16.94 GiB/s |
101
+ | | 4 KiB | decode | **24.70 GiB/s** | 4.06 GiB/s | 15.73 GiB/s |
102
+ | | 1 MiB | encode | **19.20 GiB/s** | 4.81 GiB/s | 18.19 GiB/s |
103
+ | | 1 MiB | decode | **28.72 GiB/s** | 3.80 GiB/s | 16.66 GiB/s |
104
+ | | 32 MiB | encode | **10.32 GiB/s** | 2.95 GiB/s | 9.97 GiB/s |
105
+ | | 32 MiB | decode | **10.56 GiB/s** | 3.04 GiB/s | 9.70 GiB/s |
106
+ | URL-safe | 4 KiB | encode | **17.95 GiB/s** | 5.40 GiB/s | 16.90 GiB/s |
107
+ | | 4 KiB | decode | **23.39 GiB/s** | 4.05 GiB/s | 15.75 GiB/s |
108
+ | | 1 MiB | encode | **19.19 GiB/s** | 4.80 GiB/s | 18.17 GiB/s |
109
+ | | 1 MiB | decode | **27.19 GiB/s** | 3.79 GiB/s | 16.76 GiB/s |
110
+ | | 32 MiB | encode | **10.33 GiB/s** | 2.98 GiB/s | 10.16 GiB/s |
111
+ | | 32 MiB | decode | **10.80 GiB/s** | 3.10 GiB/s | 9.36 GiB/s |
112
+
113
+ ## MurmurHash3: Rust
114
+
115
+ | Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
116
+ | --- | --- | ---: | ---: | ---: | ---: | ---: |
117
+ | x86 32-bit | 4 KiB | **4.03 GiB/s** | 2.44 GiB/s | 3.82 GiB/s | n/a | 4.02 GiB/s |
118
+ | | 1 MiB | **3.99 GiB/s** | 2.43 GiB/s | 3.79 GiB/s | n/a | 3.97 GiB/s |
119
+ | | 32 MiB | **3.98 GiB/s** | 2.42 GiB/s | 3.65 GiB/s | n/a | 3.81 GiB/s |
120
+ | x86 128-bit | 4 KiB | **8.90 GiB/s** | 4.66 GiB/s | 8.07 GiB/s | n/a | n/a |
121
+ | | 1 MiB | **9.21 GiB/s** | 4.71 GiB/s | 8.22 GiB/s | n/a | n/a |
122
+ | | 32 MiB | **9.12 GiB/s** | 4.55 GiB/s | 5.73 GiB/s | n/a | n/a |
123
+ | x64 128-bit | 4 KiB | **10.08 GiB/s** | 6.38 GiB/s | 8.85 GiB/s | 9.41 GiB/s | 8.91 GiB/s |
124
+ | | 1 MiB | **10.11 GiB/s** | 6.44 GiB/s | 8.87 GiB/s | 9.37 GiB/s | 8.88 GiB/s |
125
+ | | 32 MiB | **9.54 GiB/s** | 5.83 GiB/s | 6.39 GiB/s | 6.99 GiB/s | 6.41 GiB/s |
126
+
127
+ ## Base64: Python
128
+
129
+ Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly into Rust without an input copy.
130
+
131
+ | Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
132
+ | --- | --- | --- | ---: | ---: | ---: |
133
+ | Standard | 4 KiB | encode | 13.73 GiB/s | 0.49 GiB/s | **14.28 GiB/s** |
134
+ | | 4 KiB | decode | **15.64 GiB/s** | 1.15 GiB/s | 8.44 GiB/s |
135
+ | | 1 MiB | encode | **3.68 GiB/s** | 0.47 GiB/s | 3.32 GiB/s |
136
+ | | 1 MiB | decode | 3.64 GiB/s | 0.89 GiB/s | **4.13 GiB/s** |
137
+ | | 32 MiB | encode | **3.75 GiB/s** | 0.47 GiB/s | 3.51 GiB/s |
138
+ | | 32 MiB | decode | 3.81 GiB/s | 0.90 GiB/s | **4.42 GiB/s** |
139
+ | URL-safe | 4 KiB | encode | **13.68 GiB/s** | 0.41 GiB/s | 1.19 GiB/s |
140
+ | | 4 KiB | decode | **12.66 GiB/s** | 0.74 GiB/s | 1.56 GiB/s |
141
+ | | 1 MiB | encode | **3.63 GiB/s** | 0.36 GiB/s | 0.97 GiB/s |
142
+ | | 1 MiB | decode | **4.06 GiB/s** | 0.57 GiB/s | 1.55 GiB/s |
143
+ | | 32 MiB | encode | **3.78 GiB/s** | 0.36 GiB/s | 0.99 GiB/s |
144
+ | | 32 MiB | decode | **4.23 GiB/s** | 0.57 GiB/s | 1.67 GiB/s |
145
+
146
+ ### Reusable Python Buffers
147
+
148
+ | Alphabet | Input | Operation | hashcodecs |
149
+ | --- | --- | --- | ---: |
150
+ | Standard | 4 KiB | encode | **15.48 GiB/s** |
151
+ | | 4 KiB | decode | **17.20 GiB/s** |
152
+ | | 1 MiB | encode | **19.18 GiB/s** |
153
+ | | 1 MiB | decode | **23.82 GiB/s** |
154
+ | | 32 MiB | encode | **10.95 GiB/s** |
155
+ | | 32 MiB | decode | **10.47 GiB/s** |
156
+ | URL-safe | 4 KiB | encode | **15.36 GiB/s** |
157
+ | | 4 KiB | decode | **13.48 GiB/s** |
158
+ | | 1 MiB | encode | **19.20 GiB/s** |
159
+ | | 1 MiB | decode | **18.25 GiB/s** |
160
+ | | 32 MiB | encode | **10.87 GiB/s** |
161
+ | | 32 MiB | decode | **9.88 GiB/s** |
162
+
163
+ ## MurmurHash3: Python
164
+
165
+ | Variant | API | Input | hashcodecs | `mmh3` |
166
+ | --- | --- | --- | ---: | ---: |
167
+ | x86 32-bit | one-shot | 4 KiB | **3.80 GiB/s** | 3.72 GiB/s |
168
+ | | | 1 MiB | **4.00 GiB/s** | 3.84 GiB/s |
169
+ | | | 32 MiB | **3.99 GiB/s** | 3.68 GiB/s |
170
+ | | incremental | 4 KiB | **3.60 GiB/s** | 3.59 GiB/s |
171
+ | | | 1 MiB | **4.00 GiB/s** | 3.83 GiB/s |
172
+ | | | 32 MiB | **3.99 GiB/s** | 3.79 GiB/s |
173
+ | x86 128-bit | one-shot | 4 KiB | 8.19 GiB/s | **8.22 GiB/s** |
174
+ | | | 1 MiB | **9.28 GiB/s** | 8.89 GiB/s |
175
+ | | | 32 MiB | **9.18 GiB/s** | 6.25 GiB/s |
176
+ | | incremental | 4 KiB | **7.05 GiB/s** | 0.79 GiB/s |
177
+ | | | 1 MiB | **9.17 GiB/s** | 0.78 GiB/s |
178
+ | | | 32 MiB | **9.12 GiB/s** | 0.79 GiB/s |
179
+ | x64 128-bit | one-shot | 4 KiB | 8.86 GiB/s | **9.48 GiB/s** |
180
+ | | | 1 MiB | 10.07 GiB/s | **10.25 GiB/s** |
181
+ | | | 32 MiB | **9.55 GiB/s** | 6.82 GiB/s |
182
+ | | incremental | 4 KiB | 7.57 GiB/s | **8.02 GiB/s** |
183
+ | | | 1 MiB | **10.07 GiB/s** | 9.18 GiB/s |
184
+ | | | 32 MiB | **9.57 GiB/s** | 7.44 GiB/s |
185
+
186
+ ## SIMD References
187
+
188
+ The SIMD implementation follows the approach described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605), with AVX-512 VBMI and AArch64 NEON backends selected automatically when available.
{hashcodecs-0.2.0 → hashcodecs-0.3.0}/hatch_build.py RENAMED
@@ -15,42 +15,42 @@ class CustomBuildHook(BuildHookInterface[Any]):
15
15
  """Build the portable ABI3 extension and give the wheel its native tag."""
16
16
 
17
17
  def initialize(self, version: str, build_data: dict[str, Any]) -> None:
18
- if self.target_name != "wheel":
18
+ if self.target_name != 'wheel':
19
19
  return
20
20
 
21
21
  root = Path(self.root)
22
22
  subprocess.run(
23
23
  [
24
- "cargo",
25
- "build",
26
- "--manifest-path",
27
- str(root / "Cargo.toml"),
28
- "--release",
29
- "--features",
30
- "extension-module",
24
+ 'cargo',
25
+ 'build',
26
+ '--manifest-path',
27
+ str(root / 'Cargo.toml'),
28
+ '--release',
29
+ '--features',
30
+ 'extension-module',
31
31
  ],
32
32
  check=True,
33
33
  cwd=root,
34
34
  )
35
35
 
36
- extension = root / "target" / "release" / self._library_name()
37
- build_data["force_include"] = {str(extension): f"hashcodecs/_hashcodecs{self._extension_suffix()}"}
38
- build_data["pure_python"] = False
39
- build_data["tag"] = self._wheel_tag()
36
+ extension = root / 'target' / 'release' / self._library_name()
37
+ build_data['force_include'] = {str(extension): f'hashcodecs/_hashcodecs{self._extension_suffix()}'}
38
+ build_data['pure_python'] = False
39
+ build_data['tag'] = self._wheel_tag()
40
40
 
41
41
  @staticmethod
42
42
  def _library_name() -> str:
43
43
  system = platform.system()
44
- if system == "Windows":
45
- return "hashcodecs.dll"
46
- if system == "Darwin":
47
- return "libhashcodecs.dylib"
48
- return "libhashcodecs.so"
44
+ if system == 'Windows':
45
+ return 'hashcodecs.dll'
46
+ if system == 'Darwin':
47
+ return 'libhashcodecs.dylib'
48
+ return 'libhashcodecs.so'
49
49
 
50
50
  @staticmethod
51
51
  def _extension_suffix() -> str:
52
- return ".pyd" if platform.system() == "Windows" else ".so"
52
+ return '.pyd' if platform.system() == 'Windows' else '.so'
53
53
 
54
54
  @staticmethod
55
55
  def _wheel_tag() -> str:
56
- return f"cp310-abi3-{next(sys_tags()).platform}"
56
+ return f'cp310-abi3-{next(sys_tags()).platform}'
{hashcodecs-0.2.0 → hashcodecs-0.3.0}/pyproject.toml RENAMED
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "hashcodecs"
7
- version = "0.2.0"
7
+ version = "0.3.0"
8
8
  description = "SIMD-accelerated Base64 and MurmurHash3 codecs"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -66,6 +66,7 @@ line-length = 119
66
66
  target-version = "py312"
67
67
 
68
68
  [tool.ruff.lint]
69
+ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
69
70
  select = [
70
71
  "E",
71
72
  "F",
@@ -79,11 +80,24 @@ select = [
79
80
  "PTH",
80
81
  "PT",
81
82
  "PYI",
83
+ "Q",
82
84
  "RET",
83
85
  "RUF",
84
86
  "SIM",
85
87
  ]
86
88
 
89
+ [tool.ruff.lint.flake8-quotes]
90
+ docstring-quotes = "double"
91
+ inline-quotes = "single"
92
+
93
+ [tool.ruff.lint.isort]
94
+ combine-as-imports = false
95
+ detect-same-package = true
96
+ force-sort-within-sections = false
97
+
98
+ [tool.ruff.format]
99
+ quote-style = "single"
100
+
87
101
  [tool.coverage.run]
88
102
  branch = true
89
103
  source = ["hashcodecs"]
hashcodecs-0.3.0/python/hashcodecs/__init__.py ADDED
@@ -0,0 +1,45 @@
1
+ """SIMD-accelerated Base64 and MurmurHash3 functions."""
2
+
3
+ from .base64 import (
4
+ b64decode,
5
+ b64decode_into,
6
+ b64encode,
7
+ b64encode_into,
8
+ standard_b64decode,
9
+ standard_b64decode_into,
10
+ standard_b64encode,
11
+ standard_b64encode_into,
12
+ urlsafe_b64decode,
13
+ urlsafe_b64decode_into,
14
+ urlsafe_b64encode,
15
+ urlsafe_b64encode_into,
16
+ )
17
+ from .murmur3 import (
18
+ murmur3_32,
19
+ murmur3_x64_128,
20
+ murmur3_x64_128_digest,
21
+ murmur3_x86_32,
22
+ murmur3_x86_128,
23
+ murmur3_x86_128_digest,
24
+ )
25
+
26
+ __all__ = [
27
+ 'b64decode',
28
+ 'b64decode_into',
29
+ 'b64encode',
30
+ 'b64encode_into',
31
+ 'murmur3_32',
32
+ 'murmur3_x64_128',
33
+ 'murmur3_x64_128_digest',
34
+ 'murmur3_x86_32',
35
+ 'murmur3_x86_128',
36
+ 'murmur3_x86_128_digest',
37
+ 'standard_b64decode',
38
+ 'standard_b64decode_into',
39
+ 'standard_b64encode',
40
+ 'standard_b64encode_into',
41
+ 'urlsafe_b64decode',
42
+ 'urlsafe_b64decode_into',
43
+ 'urlsafe_b64encode',
44
+ 'urlsafe_b64encode_into',
45
+ ]
{hashcodecs-0.2.0 → hashcodecs-0.3.0}/python/hashcodecs/_hashcodecs.pyi RENAMED
@@ -1,11 +1,18 @@
1
1
  from _typeshed import ReadableBuffer
2
2
 
3
3
  def b64encode(s: ReadableBuffer, altchars: ReadableBuffer | None = None) -> bytes: ...
4
+ def b64encode_into(s: ReadableBuffer, output: bytearray, altchars: ReadableBuffer | None = None) -> int: ...
4
5
  def b64decode(
5
6
  s: str | ReadableBuffer,
6
7
  altchars: str | ReadableBuffer | None = None,
7
8
  validate: bool = False,
8
9
  ) -> bytes: ...
10
+ def b64decode_into(
11
+ s: str | ReadableBuffer,
12
+ output: bytearray,
13
+ altchars: str | ReadableBuffer | None = None,
14
+ validate: bool = False,
15
+ ) -> int: ...
9
16
  def murmur3_32(s: ReadableBuffer, seed: int = 0) -> int: ...
10
17
  def murmur3_x86_128_digest(s: ReadableBuffer, seed: int = 0) -> bytes: ...
11
18
  def murmur3_x64_128_digest(s: ReadableBuffer, seed: int = 0) -> bytes: ...
hashcodecs-0.3.0/python/hashcodecs/base64.py ADDED
@@ -0,0 +1,59 @@
1
+ """A fast, API-compatible subset of Python's :mod:`base64` module."""
2
+
3
+ from ._hashcodecs import b64decode, b64decode_into, b64encode, b64encode_into
4
+
5
+
6
+ def standard_b64encode(s) -> bytes:
7
+ """Encode *s* with the standard Base64 alphabet."""
8
+ return b64encode(s)
9
+
10
+
11
+ def standard_b64encode_into(s, output: bytearray) -> int:
12
+ """Encode *s* with the standard Base64 alphabet into *output*."""
13
+ return b64encode_into(s, output)
14
+
15
+
16
+ def standard_b64decode(s) -> bytes:
17
+ """Decode *s* with the standard Base64 alphabet."""
18
+ return b64decode(s)
19
+
20
+
21
+ def standard_b64decode_into(s, output: bytearray) -> int:
22
+ """Decode standard Base64 *s* into *output*."""
23
+ return b64decode_into(s, output)
24
+
25
+
26
+ def urlsafe_b64encode(s) -> bytes:
27
+ """Encode *s* with the URL-safe Base64 alphabet."""
28
+ return b64encode(s, b'-_')
29
+
30
+
31
+ def urlsafe_b64encode_into(s, output: bytearray) -> int:
32
+ """Encode *s* with the URL-safe Base64 alphabet into *output*."""
33
+ return b64encode_into(s, output, b'-_')
34
+
35
+
36
+ def urlsafe_b64decode(s) -> bytes:
37
+ """Decode *s* with the URL-safe Base64 alphabet."""
38
+ return b64decode(s, b'-_')
39
+
40
+
41
+ def urlsafe_b64decode_into(s, output: bytearray) -> int:
42
+ """Decode URL-safe Base64 *s* into *output*."""
43
+ return b64decode_into(s, output, b'-_')
44
+
45
+
46
+ __all__ = [
47
+ 'b64decode',
48
+ 'b64decode_into',
49
+ 'b64encode',
50
+ 'b64encode_into',
51
+ 'standard_b64decode',
52
+ 'standard_b64decode_into',
53
+ 'standard_b64encode',
54
+ 'standard_b64encode_into',
55
+ 'urlsafe_b64decode',
56
+ 'urlsafe_b64decode_into',
57
+ 'urlsafe_b64encode',
58
+ 'urlsafe_b64encode_into',
59
+ ]