hashcodecs 0.3.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. hashcodecs-0.5.0/BENCHMARK.md +113 -0
  2. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/Cargo.lock +2 -1
  3. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/Cargo.toml +3 -2
  4. hashcodecs-0.5.0/PKG-INFO +193 -0
  5. hashcodecs-0.5.0/README.md +173 -0
  6. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/pyproject.toml +8 -1
  7. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/python/hashcodecs/__init__.py +24 -0
  8. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/python/hashcodecs/_hashcodecs.pyi +50 -4
  9. hashcodecs-0.5.0/python/hashcodecs/base64.py +101 -0
  10. hashcodecs-0.5.0/python/hashcodecs/base64.pyi +69 -0
  11. hashcodecs-0.5.0/src/base64/aarch64.rs +927 -0
  12. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/base64/dispatch.rs +71 -8
  13. hashcodecs-0.5.0/src/base64/x86.rs +822 -0
  14. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/base64/x86_avx512.rs +14 -39
  15. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/base64.rs +166 -5
  16. hashcodecs-0.5.0/src/python/base64/encode.rs +167 -0
  17. hashcodecs-0.5.0/src/python/base64.rs +1310 -0
  18. hashcodecs-0.5.0/src/python/buffer.rs +183 -0
  19. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/python/murmur3.rs +28 -49
  20. hashcodecs-0.5.0/src/python.rs +52 -0
  21. hashcodecs-0.3.0/PKG-INFO +0 -202
  22. hashcodecs-0.3.0/README.md +0 -188
  23. hashcodecs-0.3.0/python/hashcodecs/base64.py +0 -59
  24. hashcodecs-0.3.0/python/hashcodecs/base64.pyi +0 -23
  25. hashcodecs-0.3.0/src/base64/aarch64.rs +0 -293
  26. hashcodecs-0.3.0/src/base64/x86.rs +0 -651
  27. hashcodecs-0.3.0/src/python/base64.rs +0 -335
  28. hashcodecs-0.3.0/src/python/buffer.rs +0 -88
  29. hashcodecs-0.3.0/src/python.rs +0 -29
  30. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/.gitignore +0 -0
  31. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/LICENSE +0 -0
  32. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/LICENSE-MIT +0 -0
  33. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/benches/base64.rs +0 -0
  34. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/benches/murmur3.rs +0 -0
  35. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/benches/support/mod.rs +0 -0
  36. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/build.rs +0 -0
  37. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/hatch_build.py +0 -0
  38. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/python/hashcodecs/murmur3.py +0 -0
  39. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/python/hashcodecs/py.typed +0 -0
  40. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/lib.rs +0 -0
  41. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/murmur3/dispatch.rs +0 -0
  42. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/murmur3/x86.rs +0 -0
  43. {hashcodecs-0.3.0 → hashcodecs-0.5.0}/src/murmur3.rs +0 -0
hashcodecs-0.5.0/BENCHMARK.md ADDED
@@ -0,0 +1,113 @@
1
+ # Benchmark Details
2
+
3
+ Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
4
+
5
+ Conditions: clean builds, one pinned logical CPU, single-threaded execution, and 15 Python samples. Higher is better.
6
+
7
+ ## Reusable Python Buffers
8
+
9
+ | Alphabet | Input | Operation | hashcodecs |
10
+ | --- | --- | --- | ---: |
11
+ | Standard | 4 KiB | encode | **15.52 GiB/s** |
12
+ | | 4 KiB | decode | **18.66 GiB/s** |
13
+ | | 1 MiB | encode | **39.91 GiB/s** |
14
+ | | 1 MiB | decode | **29.32 GiB/s** |
15
+ | | 32 MiB | encode | **10.92 GiB/s** |
16
+ | | 32 MiB | decode | **10.40 GiB/s** |
17
+ | URL-safe | 4 KiB | encode | **15.11 GiB/s** |
18
+ | | 4 KiB | decode | **14.43 GiB/s** |
19
+ | | 1 MiB | encode | **39.90 GiB/s** |
20
+ | | 1 MiB | decode | **20.69 GiB/s** |
21
+ | | 32 MiB | encode | **10.89 GiB/s** |
22
+ | | 32 MiB | decode | **10.46 GiB/s** |
23
+
24
+ ## Python Memoryview Inputs
25
+
26
+ Full immutable memoryviews reuse their underlying bytes for inputs of at least 64 KiB.
27
+
28
+ | Input | Operation | Returned `bytes` | Reusable `bytearray` |
29
+ | --- | --- | ---: | ---: |
30
+ | 4 KiB | encode | 8.20 GiB/s | **8.66 GiB/s** |
31
+ | | decode | 9.77 GiB/s | **10.30 GiB/s** |
32
+ | 1 MiB | encode | 3.48 GiB/s | **39.67 GiB/s** |
33
+ | | decode | 4.28 GiB/s | **29.21 GiB/s** |
34
+ | 32 MiB | encode | 2.98 GiB/s | **11.18 GiB/s** |
35
+ | | decode | 3.50 GiB/s | **10.49 GiB/s** |
36
+
37
+ ## Python Base64 Batches
38
+
39
+ Each value is total input throughput; the parenthesized figure is items per second. Higher is better.
40
+
41
+ | Item | Batch | Operation | hashcodecs | hashcodecs loop | pybase64 loop | CPython loop |
42
+ | ---: | ---: | --- | ---: | ---: | ---: | ---: |
43
+ | 16 B | 8 | encode | **0.38 GiB/s (25.36M)** | 0.25 GiB/s (16.81M) | 0.12 GiB/s (8.22M) | 0.19 GiB/s (12.45M) |
44
+ | | | decode | **0.35 GiB/s (23.69M)** | 0.20 GiB/s (13.27M) | 0.08 GiB/s (5.35M) | 0.12 GiB/s (7.95M) |
45
+ | 16 B | 64 | encode | **0.45 GiB/s (30.25M)** | 0.28 GiB/s (18.62M) | 0.14 GiB/s (9.17M) | 0.21 GiB/s (13.78M) |
46
+ | | | decode | **0.47 GiB/s (31.62M)** | 0.23 GiB/s (15.41M) | 0.09 GiB/s (5.96M) | 0.13 GiB/s (8.87M) |
47
+ | 16 B | 1,024 | encode | **0.48 GiB/s (32.40M)** | 0.27 GiB/s (18.38M) | 0.14 GiB/s (9.09M) | 0.20 GiB/s (13.14M) |
48
+ | | | decode | **0.48 GiB/s (32.06M)** | 0.23 GiB/s (15.15M) | 0.08 GiB/s (5.69M) | 0.13 GiB/s (8.39M) |
49
+ | 256 B | 8 | encode | 1.69 GiB/s (7.07M) | 1.42 GiB/s (5.98M) | **1.85 GiB/s (7.76M)** | 0.40 GiB/s (1.68M) |
50
+ | | | decode | **4.98 GiB/s (20.89M)** | 2.91 GiB/s (12.20M) | 1.18 GiB/s (4.93M) | 0.74 GiB/s (3.09M) |
51
+ | 256 B | 64 | encode | 1.84 GiB/s (7.73M) | 1.47 GiB/s (6.17M) | **1.93 GiB/s (8.10M)** | 0.39 GiB/s (1.64M) |
52
+ | | | decode | **6.09 GiB/s (25.52M)** | 3.17 GiB/s (13.31M) | 1.28 GiB/s (5.35M) | 0.78 GiB/s (3.28M) |
53
+ | 256 B | 1,024 | encode | 1.78 GiB/s (7.47M) | 1.50 GiB/s (6.27M) | **1.87 GiB/s (7.83M)** | 0.37 GiB/s (1.55M) |
54
+ | | | decode | **5.75 GiB/s (24.10M)** | 2.99 GiB/s (12.55M) | 1.23 GiB/s (5.17M) | 0.72 GiB/s (3.04M) |
55
+ | 4 KiB | 8 | encode | **14.24 GiB/s (3.73M)** | 12.84 GiB/s (3.37M) | 12.27 GiB/s (3.22M) | 0.46 GiB/s (0.12M) |
56
+ | | | decode | **18.68 GiB/s (4.90M)** | 15.88 GiB/s (4.16M) | 7.86 GiB/s (2.06M) | 1.14 GiB/s (0.30M) |
57
+ | 4 KiB | 64 | encode | **12.37 GiB/s (3.24M)** | 11.32 GiB/s (2.97M) | 9.57 GiB/s (2.51M) | 0.48 GiB/s (0.13M) |
58
+ | | | decode | **19.14 GiB/s (5.02M)** | 16.16 GiB/s (4.24M) | 8.09 GiB/s (2.12M) | 1.13 GiB/s (0.30M) |
59
+ | 4 KiB | 1,024 | encode | 2.03 GiB/s (0.53M) | **2.46 GiB/s (0.65M)** | 2.35 GiB/s (0.62M) | 0.41 GiB/s (0.11M) |
60
+ | | | decode | **13.93 GiB/s (3.65M)** | 12.14 GiB/s (3.18M) | 7.13 GiB/s (1.87M) | 1.11 GiB/s (0.29M) |
61
+
62
+ ## Reusable Python Base64 Batch Buffers
63
+
64
+ Each value is total input throughput for the `*_batch_into` APIs with one reusable `bytearray` per item.
65
+
66
+ | Item | Batch | Encode | Decode |
67
+ | ---: | ---: | ---: | ---: |
68
+ | 16 B | 8 | 0.33 GiB/s | 0.30 GiB/s |
69
+ | | 64 | 0.47 GiB/s | 0.44 GiB/s |
70
+ | | 1,024 | 0.47 GiB/s | 0.45 GiB/s |
71
+ | 256 B | 8 | 1.57 GiB/s | 4.01 GiB/s |
72
+ | | 64 | 1.73 GiB/s | 5.51 GiB/s |
73
+ | | 1,024 | 1.73 GiB/s | 5.29 GiB/s |
74
+ | 4 KiB | 8 | 15.39 GiB/s | 20.29 GiB/s |
75
+ | | 64 | 16.28 GiB/s | 22.12 GiB/s |
76
+ | | 1,024 | 12.95 GiB/s | 16.67 GiB/s |
77
+
78
+ ## Large Python Base64 Batches
79
+
80
+ Each value is total input throughput for 1 MiB items.
81
+
82
+ | Batch | Returned encode | Reusable encode | Returned decode | Reusable decode |
83
+ | ---: | ---: | ---: | ---: | ---: |
84
+ | 1 | 3.52 GiB/s | **39.63 GiB/s** | 3.81 GiB/s | **29.12 GiB/s** |
85
+ | 2 | 3.31 GiB/s | **20.90 GiB/s** | 3.99 GiB/s | **21.50 GiB/s** |
86
+ | 4 | 3.19 GiB/s | **20.15 GiB/s** | 3.91 GiB/s | **21.00 GiB/s** |
87
+ | 8 | 3.17 GiB/s | **19.45 GiB/s** | 3.87 GiB/s | **18.99 GiB/s** |
88
+ | 16 | 2.97 GiB/s | **12.96 GiB/s** | 3.68 GiB/s | **12.18 GiB/s** |
89
+ | 32 | 2.88 GiB/s | **11.09 GiB/s** | 3.29 GiB/s | **10.60 GiB/s** |
90
+
91
+ ## Mutable Python Inputs
92
+
93
+ ### Base64
94
+
95
+ | Input | Operation | Returned `bytes` | Reusable `bytearray` |
96
+ | --- | --- | ---: | ---: |
97
+ | 4 KiB | encode | 13.90 GiB/s | **15.22 GiB/s** |
98
+ | | decode | 16.57 GiB/s | **18.27 GiB/s** |
99
+ | 1 MiB | encode | 3.13 GiB/s | **39.59 GiB/s** |
100
+ | | decode | 3.67 GiB/s | **29.24 GiB/s** |
101
+ | 32 MiB | encode | 2.99 GiB/s | **11.20 GiB/s** |
102
+ | | decode | 3.65 GiB/s | **10.70 GiB/s** |
103
+
104
+ ### MurmurHash3
105
+
106
+ | Variant | API | 4 KiB | 1 MiB | 32 MiB |
107
+ | --- | --- | ---: | ---: | ---: |
108
+ | x86 32-bit | one-shot | **3.79 GiB/s** | **3.98 GiB/s** | **3.97 GiB/s** |
109
+ | | incremental | **3.58 GiB/s** | **3.96 GiB/s** | **3.92 GiB/s** |
110
+ | x86 128-bit | one-shot | **8.09 GiB/s** | **9.22 GiB/s** | **9.12 GiB/s** |
111
+ | | incremental | **7.02 GiB/s** | **9.17 GiB/s** | **9.01 GiB/s** |
112
+ | x64 128-bit | one-shot | **8.68 GiB/s** | **10.10 GiB/s** | **9.54 GiB/s** |
113
+ | | incremental | **7.70 GiB/s** | **10.01 GiB/s** | **9.41 GiB/s** |
{hashcodecs-0.3.0 → hashcodecs-0.5.0}/Cargo.lock RENAMED
@@ -194,12 +194,13 @@ dependencies = [
194
194
 
195
195
  [[package]]
196
196
  name = "hashcodecs"
197
- version = "0.3.0"
197
+ version = "0.5.0"
198
198
  dependencies = [
199
199
  "base64",
200
200
  "base64-turbo",
201
201
  "criterion",
202
202
  "fastmurmur3",
203
+ "memchr",
203
204
  "mimalloc",
204
205
  "mm3h",
205
206
  "murmur3",
{hashcodecs-0.3.0 → hashcodecs-0.5.0}/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "hashcodecs"
3
- version = "0.3.0"
3
+ version = "0.5.0"
4
4
  edition = "2024"
5
5
  rust-version = "1.89"
6
6
  description = "SIMD-accelerated Base64 codecs and fast MurmurHash3 implementations"
@@ -16,10 +16,11 @@ name = "hashcodecs"
16
16
  crate-type = ["rlib", "cdylib"]
17
17
 
18
18
  [features]
19
- python = ["dep:pyo3"]
19
+ python = ["dep:memchr", "dep:pyo3"]
20
20
  extension-module = ["python", "pyo3/extension-module"]
21
21
 
22
22
  [dependencies]
23
+ memchr = { version = "2.8.0", optional = true }
23
24
  mimalloc = "0.1.52"
24
25
  pyo3 = { version = "0.29.2", features = ["abi3-py310"], optional = true }
25
26
 
hashcodecs-0.5.0/PKG-INFO ADDED
@@ -0,0 +1,193 @@
1
+ Metadata-Version: 2.5
2
+ Name: hashcodecs
3
+ Version: 0.5.0
4
+ Summary: SIMD-accelerated Base64 and MurmurHash3 codecs
5
+ License-Expression: MIT OR Apache-2.0
6
+ License-File: LICENSE
7
+ License-File: LICENSE-MIT
8
+ Classifier: Operating System :: OS Independent
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Programming Language :: Python :: 3.14
15
+ Classifier: Programming Language :: Python :: 3.15
16
+ Classifier: Programming Language :: Rust
17
+ Classifier: Topic :: Software Development :: Libraries
18
+ Requires-Python: >=3.10
19
+ Description-Content-Type: text/markdown
20
+
21
+ # hashcodecs
22
+
23
+ [![CI](https://img.shields.io/github/actions/workflow/status/kozistr/hashcodecs-rs/ci.yml?branch=main&style=for-the-badge&logo=github)](https://github.com/kozistr/hashcodecs-rs/actions/workflows/ci.yml)
24
+ [![Codecov](https://codecov.io/gh/kozistr/hashcodecs-rs/graph/badge.svg)](https://app.codecov.io/gh/kozistr/hashcodecs-rs)
25
+ [![PyPI](https://img.shields.io/pypi/v/hashcodecs?style=for-the-badge&logo=pypi)](https://pypi.org/project/hashcodecs/)
26
+ [![Python](https://img.shields.io/pypi/pyversions/hashcodecs?style=for-the-badge&logo=python)](https://pypi.org/project/hashcodecs/)
27
+ [![License](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-brightgreen?style=for-the-badge)](https://github.com/kozistr/hashcodecs-rs#license)
28
+ [![Downloads](https://img.shields.io/pypi/dm/hashcodecs?style=for-the-badge&label=downloads)](https://pypi.org/project/hashcodecs/)
29
+
30
+ `hashcodecs` provides runtime-dispatched SIMD Base64 codecs and fast, reference-compatible MurmurHash3 functions for Rust and Python.
31
+
32
+ ## Design
33
+
34
+ - Runtime-dispatched SIMD Base64 with portable scalar fallbacks.
35
+ - Reference-compatible MurmurHash3 with SIMD acceleration.
36
+ - Rust and Python `*_into` APIs for caller-managed output buffers.
37
+ - A familiar Python Base64 API, including native batch encode and decode operations.
38
+
39
+ ## Install
40
+
41
+ ```
42
+ pip3 install hashcodecs
43
+ ```
44
+
45
+ ## Usage
46
+
47
+ ### Rust
48
+
49
+ ```rust
50
+ let encoded = hashcodecs::b64encode(b"hello");
51
+ assert_eq!(encoded, "aGVsbG8=");
52
+
53
+ let mut output = [0_u8; 8];
54
+ let written = hashcodecs::b64encode_into(b"hello", &mut output).unwrap();
55
+ assert_eq!(&output[..written], b"aGVsbG8=");
56
+ assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
57
+ ```
58
+
59
+ ### Python
60
+
61
+ ```python
62
+ import hashcodecs.base64 as base64
63
+ from hashcodecs import murmur3_32, murmur3_x64_128
64
+
65
+ assert base64.b64encode(b'hello') == b'aGVsbG8='
66
+ assert base64.b64decode(b'aGVsbG8=') == b'hello'
67
+ assert base64.b64encode_batch([b'hello', b'world']) == [b'aGVsbG8=', b'd29ybGQ=']
68
+ assert base64.b64decode_batch([b'aGVsbG8=', 'd29ybGQ=']) == [b'hello', b'world']
69
+ assert base64.b64encode(b'hello', padded=False) == b'aGVsbG8'
70
+ assert base64.b64decode(b'aGVsbG8', padded=False, canonical=True) == b'hello'
71
+ assert murmur3_32(b'hello') == 0x248BFA47
72
+
73
+ payload = b'hello'
74
+ encoded = bytearray(4 * ((len(payload) + 2) // 3))
75
+ encoded_len = base64.b64encode_into(payload, encoded)
76
+ decoded = bytearray(len(encoded))
77
+ decoded_len = base64.b64decode_into(encoded, decoded, validate=True)
78
+ assert encoded[:encoded_len] == b'aGVsbG8='
79
+ assert decoded[:decoded_len] == payload
80
+
81
+ hasher = murmur3_x64_128(seed=42)
82
+ hasher.update(b'hello')
83
+ assert hasher.hexdigest() == hasher.digest().hex()
84
+ ```
85
+
86
+ Build an installable wheel and source distribution with:
87
+
88
+ ```sh
89
+ uv build
90
+ ```
91
+
92
+ # Benchmark
93
+
94
+ Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
95
+
96
+ Conditions: clean builds, one pinned logical CPU, single-threaded execution, 50 Rust samples, and 15 Python samples. Returned-output allocation is included except in the reusable-buffer table. Higher is better.
97
+
98
+ ## Run Locally
99
+
100
+ Comparison crates are development-only dependencies and are not included in consumer builds.
101
+
102
+ ```sh
103
+ cargo bench --bench base64
104
+ cargo bench --bench murmur3
105
+ uv sync --group benchmark --no-install-project
106
+ uv run --no-project --with . python benchmarks/python_base64.py
107
+ uv run --no-project --with . python benchmarks/python_base64.py --into
108
+ uv run --no-project --with . python benchmarks/python_base64.py --bytearray-input
109
+ uv run --no-project --with . python benchmarks/python_base64.py --memoryview-input
110
+ uv run --no-project --with . python benchmarks/python_base64_batch.py
111
+ uv run --no-project --with . python benchmarks/python_base64_batch.py --large
112
+ uv run --no-project --with . python benchmarks/python_murmur3.py
113
+ uv run --no-project --with . python benchmarks/python_murmur3.py --incremental
114
+ ```
115
+
116
+ ## Base64: Rust
117
+
118
+ | Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
119
+ | --- | --- | --- | ---: | ---: | ---: |
120
+ | Standard | 4 KiB | encode | **20.41 GiB/s** | 5.81 GiB/s | 18.29 GiB/s |
121
+ | | 4 KiB | decode | **27.11 GiB/s** | 4.33 GiB/s | 16.95 GiB/s |
122
+ | | 1 MiB | encode | **42.38 GiB/s** | 5.16 GiB/s | 19.60 GiB/s |
123
+ | | 1 MiB | decode | **31.07 GiB/s** | 4.10 GiB/s | 18.17 GiB/s |
124
+ | | 32 MiB | encode | **11.97 GiB/s** | 3.28 GiB/s | 10.94 GiB/s |
125
+ | | 32 MiB | decode | **11.51 GiB/s** | 3.35 GiB/s | 10.45 GiB/s |
126
+ | URL-safe | 4 KiB | encode | **20.46 GiB/s** | 5.81 GiB/s | 18.31 GiB/s |
127
+ | | 4 KiB | decode | **25.65 GiB/s** | 4.35 GiB/s | 16.96 GiB/s |
128
+ | | 1 MiB | encode | **42.56 GiB/s** | 5.18 GiB/s | 19.66 GiB/s |
129
+ | | 1 MiB | decode | **29.42 GiB/s** | 4.08 GiB/s | 18.16 GiB/s |
130
+ | | 32 MiB | encode | **11.86 GiB/s** | 3.29 GiB/s | 11.09 GiB/s |
131
+ | | 32 MiB | decode | **11.67 GiB/s** | 3.36 GiB/s | 10.62 GiB/s |
132
+
133
+ ## MurmurHash3: Rust
134
+
135
+ | Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
136
+ | --- | --- | ---: | ---: | ---: | ---: | ---: |
137
+ | x86 32-bit | 4 KiB | **4.30 GiB/s** | 2.60 GiB/s | 4.09 GiB/s | n/a | **4.30 GiB/s** |
138
+ | | 1 MiB | **4.26 GiB/s** | 2.59 GiB/s | 4.04 GiB/s | n/a | 4.25 GiB/s |
139
+ | | 32 MiB | **4.21 GiB/s** | 2.54 GiB/s | 3.87 GiB/s | n/a | 4.03 GiB/s |
140
+ | x86 128-bit | 4 KiB | **9.54 GiB/s** | 4.97 GiB/s | 8.64 GiB/s | n/a | n/a |
141
+ | | 1 MiB | **9.84 GiB/s** | 5.06 GiB/s | 8.77 GiB/s | n/a | n/a |
142
+ | | 32 MiB | **9.63 GiB/s** | 4.84 GiB/s | 6.05 GiB/s | n/a | n/a |
143
+ | x64 128-bit | 4 KiB | **10.76 GiB/s** | 6.83 GiB/s | 9.45 GiB/s | 10.06 GiB/s | 9.54 GiB/s |
144
+ | | 1 MiB | **10.84 GiB/s** | 6.89 GiB/s | 9.51 GiB/s | 10.04 GiB/s | 9.51 GiB/s |
145
+ | | 32 MiB | **10.12 GiB/s** | 6.11 GiB/s | 6.64 GiB/s | 7.25 GiB/s | 6.70 GiB/s |
146
+
147
+ ## Base64: Python
148
+
149
+ Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly into Rust without an input copy.
150
+
151
+ | Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
152
+ | --- | --- | --- | ---: | ---: | ---: |
153
+ | Standard | 4 KiB | encode | **14.42 GiB/s** | 0.49 GiB/s | 14.36 GiB/s |
154
+ | | 4 KiB | decode | **16.72 GiB/s** | 1.12 GiB/s | 8.35 GiB/s |
155
+ | | 1 MiB | encode | **3.69 GiB/s** | 0.43 GiB/s | 3.51 GiB/s |
156
+ | | 1 MiB | decode | **4.90 GiB/s** | 0.97 GiB/s | 4.60 GiB/s |
157
+ | | 32 MiB | encode | **2.81 GiB/s** | 0.44 GiB/s | 2.60 GiB/s |
158
+ | | 32 MiB | decode | 3.15 GiB/s | 0.93 GiB/s | **3.49 GiB/s** |
159
+ | URL-safe | 4 KiB | encode | **14.48 GiB/s** | 0.40 GiB/s | 1.19 GiB/s |
160
+ | | 4 KiB | decode | **13.45 GiB/s** | 0.75 GiB/s | 1.56 GiB/s |
161
+ | | 1 MiB | encode | **3.96 GiB/s** | 0.36 GiB/s | 0.92 GiB/s |
162
+ | | 1 MiB | decode | **4.53 GiB/s** | 0.61 GiB/s | 1.58 GiB/s |
163
+ | | 32 MiB | encode | **2.64 GiB/s** | 0.34 GiB/s | 0.83 GiB/s |
164
+ | | 32 MiB | decode | **3.15 GiB/s** | 0.60 GiB/s | 1.49 GiB/s |
165
+
166
+ ## MurmurHash3: Python
167
+
168
+ | Variant | API | Input | hashcodecs | `mmh3` |
169
+ | --- | --- | --- | ---: | ---: |
170
+ | x86 32-bit | one-shot | 4 KiB | **3.79 GiB/s** | 3.71 GiB/s |
171
+ | | | 1 MiB | **3.98 GiB/s** | 3.83 GiB/s |
172
+ | | | 32 MiB | **3.97 GiB/s** | 3.66 GiB/s |
173
+ | | incremental | 4 KiB | **3.58 GiB/s** | **3.58 GiB/s** |
174
+ | | | 1 MiB | **3.96 GiB/s** | 3.81 GiB/s |
175
+ | | | 32 MiB | **3.92 GiB/s** | 3.74 GiB/s |
176
+ | x86 128-bit | one-shot | 4 KiB | 8.09 GiB/s | **8.21 GiB/s** |
177
+ | | | 1 MiB | **9.22 GiB/s** | 8.87 GiB/s |
178
+ | | | 32 MiB | **9.12 GiB/s** | 6.15 GiB/s |
179
+ | | incremental | 4 KiB | **7.02 GiB/s** | 0.77 GiB/s |
180
+ | | | 1 MiB | **9.17 GiB/s** | 0.80 GiB/s |
181
+ | | | 32 MiB | **9.01 GiB/s** | 0.79 GiB/s |
182
+ | x64 128-bit | one-shot | 4 KiB | 8.68 GiB/s | **9.46 GiB/s** |
183
+ | | | 1 MiB | 10.10 GiB/s | **10.24 GiB/s** |
184
+ | | | 32 MiB | **9.54 GiB/s** | 6.61 GiB/s |
185
+ | | incremental | 4 KiB | 7.70 GiB/s | **7.99 GiB/s** |
186
+ | | | 1 MiB | **10.01 GiB/s** | 9.28 GiB/s |
187
+ | | | 32 MiB | **9.41 GiB/s** | 7.34 GiB/s |
188
+
189
+ Reusable-buffer and mutable-input results are available in [BENCHMARK.md](https://github.com/kozistr/hashcodecs-rs/blob/main/BENCHMARK.md).
190
+
191
+ ## SIMD References
192
+
193
+ The SIMD implementation follows the approach described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605), with AVX-512 VBMI and AArch64 NEON backends selected automatically when available.
hashcodecs-0.5.0/README.md ADDED
@@ -0,0 +1,173 @@
1
+ # hashcodecs
2
+
3
+ [![CI](https://img.shields.io/github/actions/workflow/status/kozistr/hashcodecs-rs/ci.yml?branch=main&style=for-the-badge&logo=github)](https://github.com/kozistr/hashcodecs-rs/actions/workflows/ci.yml)
4
+ [![Codecov](https://codecov.io/gh/kozistr/hashcodecs-rs/graph/badge.svg)](https://app.codecov.io/gh/kozistr/hashcodecs-rs)
5
+ [![PyPI](https://img.shields.io/pypi/v/hashcodecs?style=for-the-badge&logo=pypi)](https://pypi.org/project/hashcodecs/)
6
+ [![Python](https://img.shields.io/pypi/pyversions/hashcodecs?style=for-the-badge&logo=python)](https://pypi.org/project/hashcodecs/)
7
+ [![License](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-brightgreen?style=for-the-badge)](https://github.com/kozistr/hashcodecs-rs#license)
8
+ [![Downloads](https://img.shields.io/pypi/dm/hashcodecs?style=for-the-badge&label=downloads)](https://pypi.org/project/hashcodecs/)
9
+
10
+ `hashcodecs` provides runtime-dispatched SIMD Base64 codecs and fast, reference-compatible MurmurHash3 functions for Rust and Python.
11
+
12
+ ## Design
13
+
14
+ - Runtime-dispatched SIMD Base64 with portable scalar fallbacks.
15
+ - Reference-compatible MurmurHash3 with SIMD acceleration.
16
+ - Rust and Python `*_into` APIs for caller-managed output buffers.
17
+ - A familiar Python Base64 API, including native batch encode and decode operations.
18
+
19
+ ## Install
20
+
21
+ ```
22
+ pip3 install hashcodecs
23
+ ```
24
+
25
+ ## Usage
26
+
27
+ ### Rust
28
+
29
+ ```rust
30
+ let encoded = hashcodecs::b64encode(b"hello");
31
+ assert_eq!(encoded, "aGVsbG8=");
32
+
33
+ let mut output = [0_u8; 8];
34
+ let written = hashcodecs::b64encode_into(b"hello", &mut output).unwrap();
35
+ assert_eq!(&output[..written], b"aGVsbG8=");
36
+ assert_eq!(hashcodecs::murmur3_x86_32(b"hello", 0), 0x248b_fa47);
37
+ ```
38
+
39
+ ### Python
40
+
41
+ ```python
42
+ import hashcodecs.base64 as base64
43
+ from hashcodecs import murmur3_32, murmur3_x64_128
44
+
45
+ assert base64.b64encode(b'hello') == b'aGVsbG8='
46
+ assert base64.b64decode(b'aGVsbG8=') == b'hello'
47
+ assert base64.b64encode_batch([b'hello', b'world']) == [b'aGVsbG8=', b'd29ybGQ=']
48
+ assert base64.b64decode_batch([b'aGVsbG8=', 'd29ybGQ=']) == [b'hello', b'world']
49
+ assert base64.b64encode(b'hello', padded=False) == b'aGVsbG8'
50
+ assert base64.b64decode(b'aGVsbG8', padded=False, canonical=True) == b'hello'
51
+ assert murmur3_32(b'hello') == 0x248BFA47
52
+
53
+ payload = b'hello'
54
+ encoded = bytearray(4 * ((len(payload) + 2) // 3))
55
+ encoded_len = base64.b64encode_into(payload, encoded)
56
+ decoded = bytearray(len(encoded))
57
+ decoded_len = base64.b64decode_into(encoded, decoded, validate=True)
58
+ assert encoded[:encoded_len] == b'aGVsbG8='
59
+ assert decoded[:decoded_len] == payload
60
+
61
+ hasher = murmur3_x64_128(seed=42)
62
+ hasher.update(b'hello')
63
+ assert hasher.hexdigest() == hasher.digest().hex()
64
+ ```
65
+
66
+ Build an installable wheel and source distribution with:
67
+
68
+ ```sh
69
+ uv build
70
+ ```
71
+
72
+ # Benchmark
73
+
74
+ Environment: Windows 10 x64 and Intel Core Ultra 7 265K.
75
+
76
+ Conditions: clean builds, one pinned logical CPU, single-threaded execution, 50 Rust samples, and 15 Python samples. Returned-output allocation is included except in the reusable-buffer table. Higher is better.
77
+
78
+ ## Run Locally
79
+
80
+ Comparison crates are development-only dependencies and are not included in consumer builds.
81
+
82
+ ```sh
83
+ cargo bench --bench base64
84
+ cargo bench --bench murmur3
85
+ uv sync --group benchmark --no-install-project
86
+ uv run --no-project --with . python benchmarks/python_base64.py
87
+ uv run --no-project --with . python benchmarks/python_base64.py --into
88
+ uv run --no-project --with . python benchmarks/python_base64.py --bytearray-input
89
+ uv run --no-project --with . python benchmarks/python_base64.py --memoryview-input
90
+ uv run --no-project --with . python benchmarks/python_base64_batch.py
91
+ uv run --no-project --with . python benchmarks/python_base64_batch.py --large
92
+ uv run --no-project --with . python benchmarks/python_murmur3.py
93
+ uv run --no-project --with . python benchmarks/python_murmur3.py --incremental
94
+ ```
95
+
96
+ ## Base64: Rust
97
+
98
+ | Alphabet | Input | Operation | hashcodecs | `base64` | `base64-turbo` |
99
+ | --- | --- | --- | ---: | ---: | ---: |
100
+ | Standard | 4 KiB | encode | **20.41 GiB/s** | 5.81 GiB/s | 18.29 GiB/s |
101
+ | | 4 KiB | decode | **27.11 GiB/s** | 4.33 GiB/s | 16.95 GiB/s |
102
+ | | 1 MiB | encode | **42.38 GiB/s** | 5.16 GiB/s | 19.60 GiB/s |
103
+ | | 1 MiB | decode | **31.07 GiB/s** | 4.10 GiB/s | 18.17 GiB/s |
104
+ | | 32 MiB | encode | **11.97 GiB/s** | 3.28 GiB/s | 10.94 GiB/s |
105
+ | | 32 MiB | decode | **11.51 GiB/s** | 3.35 GiB/s | 10.45 GiB/s |
106
+ | URL-safe | 4 KiB | encode | **20.46 GiB/s** | 5.81 GiB/s | 18.31 GiB/s |
107
+ | | 4 KiB | decode | **25.65 GiB/s** | 4.35 GiB/s | 16.96 GiB/s |
108
+ | | 1 MiB | encode | **42.56 GiB/s** | 5.18 GiB/s | 19.66 GiB/s |
109
+ | | 1 MiB | decode | **29.42 GiB/s** | 4.08 GiB/s | 18.16 GiB/s |
110
+ | | 32 MiB | encode | **11.86 GiB/s** | 3.29 GiB/s | 11.09 GiB/s |
111
+ | | 32 MiB | decode | **11.67 GiB/s** | 3.36 GiB/s | 10.62 GiB/s |
112
+
113
+ ## MurmurHash3: Rust
114
+
115
+ | Variant | Input | hashcodecs | `murmur3` | `murmurs` | `fastmurmur3` | `mm3h` |
116
+ | --- | --- | ---: | ---: | ---: | ---: | ---: |
117
+ | x86 32-bit | 4 KiB | **4.30 GiB/s** | 2.60 GiB/s | 4.09 GiB/s | n/a | **4.30 GiB/s** |
118
+ | | 1 MiB | **4.26 GiB/s** | 2.59 GiB/s | 4.04 GiB/s | n/a | 4.25 GiB/s |
119
+ | | 32 MiB | **4.21 GiB/s** | 2.54 GiB/s | 3.87 GiB/s | n/a | 4.03 GiB/s |
120
+ | x86 128-bit | 4 KiB | **9.54 GiB/s** | 4.97 GiB/s | 8.64 GiB/s | n/a | n/a |
121
+ | | 1 MiB | **9.84 GiB/s** | 5.06 GiB/s | 8.77 GiB/s | n/a | n/a |
122
+ | | 32 MiB | **9.63 GiB/s** | 4.84 GiB/s | 6.05 GiB/s | n/a | n/a |
123
+ | x64 128-bit | 4 KiB | **10.76 GiB/s** | 6.83 GiB/s | 9.45 GiB/s | 10.06 GiB/s | 9.54 GiB/s |
124
+ | | 1 MiB | **10.84 GiB/s** | 6.89 GiB/s | 9.51 GiB/s | 10.04 GiB/s | 9.51 GiB/s |
125
+ | | 32 MiB | **10.12 GiB/s** | 6.11 GiB/s | 6.64 GiB/s | 7.25 GiB/s | 6.70 GiB/s |
126
+
127
+ ## Base64: Python
128
+
129
+ Python decoding uses `validate=True`, and `hashcodecs` passes `bytes` directly into Rust without an input copy.
130
+
131
+ | Alphabet | Input | Operation | hashcodecs | CPython `base64` | `pybase64` |
132
+ | --- | --- | --- | ---: | ---: | ---: |
133
+ | Standard | 4 KiB | encode | **14.42 GiB/s** | 0.49 GiB/s | 14.36 GiB/s |
134
+ | | 4 KiB | decode | **16.72 GiB/s** | 1.12 GiB/s | 8.35 GiB/s |
135
+ | | 1 MiB | encode | **3.69 GiB/s** | 0.43 GiB/s | 3.51 GiB/s |
136
+ | | 1 MiB | decode | **4.90 GiB/s** | 0.97 GiB/s | 4.60 GiB/s |
137
+ | | 32 MiB | encode | **2.81 GiB/s** | 0.44 GiB/s | 2.60 GiB/s |
138
+ | | 32 MiB | decode | 3.15 GiB/s | 0.93 GiB/s | **3.49 GiB/s** |
139
+ | URL-safe | 4 KiB | encode | **14.48 GiB/s** | 0.40 GiB/s | 1.19 GiB/s |
140
+ | | 4 KiB | decode | **13.45 GiB/s** | 0.75 GiB/s | 1.56 GiB/s |
141
+ | | 1 MiB | encode | **3.96 GiB/s** | 0.36 GiB/s | 0.92 GiB/s |
142
+ | | 1 MiB | decode | **4.53 GiB/s** | 0.61 GiB/s | 1.58 GiB/s |
143
+ | | 32 MiB | encode | **2.64 GiB/s** | 0.34 GiB/s | 0.83 GiB/s |
144
+ | | 32 MiB | decode | **3.15 GiB/s** | 0.60 GiB/s | 1.49 GiB/s |
145
+
146
+ ## MurmurHash3: Python
147
+
148
+ | Variant | API | Input | hashcodecs | `mmh3` |
149
+ | --- | --- | --- | ---: | ---: |
150
+ | x86 32-bit | one-shot | 4 KiB | **3.79 GiB/s** | 3.71 GiB/s |
151
+ | | | 1 MiB | **3.98 GiB/s** | 3.83 GiB/s |
152
+ | | | 32 MiB | **3.97 GiB/s** | 3.66 GiB/s |
153
+ | | incremental | 4 KiB | **3.58 GiB/s** | **3.58 GiB/s** |
154
+ | | | 1 MiB | **3.96 GiB/s** | 3.81 GiB/s |
155
+ | | | 32 MiB | **3.92 GiB/s** | 3.74 GiB/s |
156
+ | x86 128-bit | one-shot | 4 KiB | 8.09 GiB/s | **8.21 GiB/s** |
157
+ | | | 1 MiB | **9.22 GiB/s** | 8.87 GiB/s |
158
+ | | | 32 MiB | **9.12 GiB/s** | 6.15 GiB/s |
159
+ | | incremental | 4 KiB | **7.02 GiB/s** | 0.77 GiB/s |
160
+ | | | 1 MiB | **9.17 GiB/s** | 0.80 GiB/s |
161
+ | | | 32 MiB | **9.01 GiB/s** | 0.79 GiB/s |
162
+ | x64 128-bit | one-shot | 4 KiB | 8.68 GiB/s | **9.46 GiB/s** |
163
+ | | | 1 MiB | 10.10 GiB/s | **10.24 GiB/s** |
164
+ | | | 32 MiB | **9.54 GiB/s** | 6.61 GiB/s |
165
+ | | incremental | 4 KiB | 7.70 GiB/s | **7.99 GiB/s** |
166
+ | | | 1 MiB | **10.01 GiB/s** | 9.28 GiB/s |
167
+ | | | 32 MiB | **9.41 GiB/s** | 7.34 GiB/s |
168
+
169
+ Reusable-buffer and mutable-input results are available in [BENCHMARK.md](https://github.com/kozistr/hashcodecs-rs/blob/main/BENCHMARK.md).
170
+
171
+ ## SIMD References
172
+
173
+ The SIMD implementation follows the approach described in [Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/abs/1704.00605), with AVX-512 VBMI and AArch64 NEON backends selected automatically when available.
{hashcodecs-0.3.0 → hashcodecs-0.5.0}/pyproject.toml RENAMED
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "hashcodecs"
7
- version = "0.3.0"
7
+ version = "0.5.0"
8
8
  description = "SIMD-accelerated Base64 and MurmurHash3 codecs"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -12,6 +12,12 @@ license = "MIT OR Apache-2.0"
12
12
  license-files = ["LICENSE", "LICENSE-MIT"]
13
13
  classifiers = [
14
14
  "Programming Language :: Python :: 3",
15
+ "Programming Language :: Python :: 3.10",
16
+ "Programming Language :: Python :: 3.11",
17
+ "Programming Language :: Python :: 3.12",
18
+ "Programming Language :: Python :: 3.13",
19
+ "Programming Language :: Python :: 3.14",
20
+ "Programming Language :: Python :: 3.15",
15
21
  "Programming Language :: Rust",
16
22
  "Operating System :: OS Independent",
17
23
  "Topic :: Software Development :: Libraries",
@@ -33,6 +39,7 @@ packages = ["python/hashcodecs"]
33
39
 
34
40
  [tool.hatch.build.targets.sdist]
35
41
  include = [
42
+ "/BENCHMARK.md",
36
43
  "/Cargo.lock",
37
44
  "/Cargo.toml",
38
45
  "/build.rs",
{hashcodecs-0.3.0 → hashcodecs-0.5.0}/python/hashcodecs/__init__.py RENAMED
@@ -2,16 +2,28 @@
2
2
 
3
3
  from .base64 import (
4
4
  b64decode,
5
+ b64decode_batch,
6
+ b64decode_batch_into,
5
7
  b64decode_into,
6
8
  b64encode,
9
+ b64encode_batch,
10
+ b64encode_batch_into,
7
11
  b64encode_into,
8
12
  standard_b64decode,
13
+ standard_b64decode_batch,
14
+ standard_b64decode_batch_into,
9
15
  standard_b64decode_into,
10
16
  standard_b64encode,
17
+ standard_b64encode_batch,
18
+ standard_b64encode_batch_into,
11
19
  standard_b64encode_into,
12
20
  urlsafe_b64decode,
21
+ urlsafe_b64decode_batch,
22
+ urlsafe_b64decode_batch_into,
13
23
  urlsafe_b64decode_into,
14
24
  urlsafe_b64encode,
25
+ urlsafe_b64encode_batch,
26
+ urlsafe_b64encode_batch_into,
15
27
  urlsafe_b64encode_into,
16
28
  )
17
29
  from .murmur3 import (
@@ -25,8 +37,12 @@ from .murmur3 import (
25
37
 
26
38
  __all__ = [
27
39
  'b64decode',
40
+ 'b64decode_batch',
41
+ 'b64decode_batch_into',
28
42
  'b64decode_into',
29
43
  'b64encode',
44
+ 'b64encode_batch',
45
+ 'b64encode_batch_into',
30
46
  'b64encode_into',
31
47
  'murmur3_32',
32
48
  'murmur3_x64_128',
@@ -35,11 +51,19 @@ __all__ = [
35
51
  'murmur3_x86_128',
36
52
  'murmur3_x86_128_digest',
37
53
  'standard_b64decode',
54
+ 'standard_b64decode_batch',
55
+ 'standard_b64decode_batch_into',
38
56
  'standard_b64decode_into',
39
57
  'standard_b64encode',
58
+ 'standard_b64encode_batch',
59
+ 'standard_b64encode_batch_into',
40
60
  'standard_b64encode_into',
41
61
  'urlsafe_b64decode',
62
+ 'urlsafe_b64decode_batch',
63
+ 'urlsafe_b64decode_batch_into',
42
64
  'urlsafe_b64decode_into',
43
65
  'urlsafe_b64encode',
66
+ 'urlsafe_b64encode_batch',
67
+ 'urlsafe_b64encode_batch_into',
44
68
  'urlsafe_b64encode_into',
45
69
  ]