fastpybase64 0.1.3__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fastpybase64-0.1.3/fastpybase64.egg-info → fastpybase64-0.2.0}/PKG-INFO +24 -2
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/README.md +23 -1
- fastpybase64-0.2.0/bench.py +196 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0/fastpybase64.egg-info}/PKG-INFO +24 -2
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/fastpybase64.egg-info/SOURCES.txt +1 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/pyproject.toml +1 -1
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/src/fastpybase64.cpp +67 -8
- fastpybase64-0.2.0/tests/conftest.py +12 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/tests/test_base64.py +22 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/vendor/simdutf.cpp +3115 -795
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/vendor/simdutf.h +390 -103
- fastpybase64-0.1.3/bench.py +0 -119
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/LICENSE-APACHE +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/LICENSE-MIT +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/MANIFEST.in +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/doc/make_charts.py +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/fastpybase64.egg-info/dependency_links.txt +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/fastpybase64.egg-info/requires.txt +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/fastpybase64.egg-info/top_level.txt +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/setup.cfg +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/setup.py +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/vendor/simdutf-LICENSE-APACHE +0 -0
- {fastpybase64-0.1.3 → fastpybase64-0.2.0}/vendor/simdutf-LICENSE-MIT +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fastpybase64
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Fast Base64 encoding and decoding for Python, built on simdutf
|
|
5
5
|
Author-email: Daniel Lemire <daniel@lemire.me>
|
|
6
6
|
License-Expression: Apache-2.0 OR MIT
|
|
@@ -81,6 +81,7 @@ base64.encodebytes(data) # MIME: lines of 76 characters
|
|
|
81
81
|
| `urlsafe_b64encode(s)`, `urlsafe_b64decode(s)` | alphabet with `-` and `_` |
|
|
82
82
|
| `encodebytes(s)`, `decodebytes(s)` | MIME style: a newline after every 76 characters |
|
|
83
83
|
| `get_implementation()` | the kernel simdutf selected for this processor (`icelake`, `haswell`, `arm64`, ...) |
|
|
84
|
+
| `set_exact_sizing(enabled)`, `get_exact_sizing()` | size decoded outputs exactly (off by default, see [Memory](#memory)) |
|
|
84
85
|
| `simdutf_version`, `__version__` | versions of simdutf and of fastpybase64 |
|
|
85
86
|
|
|
86
87
|
The functions accept what `base64` accepts: `bytes`, `bytearray`,
|
|
@@ -116,6 +117,27 @@ the same message, and the rare inputs where `base64` is lenient in odd ways
|
|
|
116
117
|
simdutf and then replacing the two characters; decoding with such an
|
|
117
118
|
alphabet is left to `base64`.
|
|
118
119
|
|
|
120
|
+
### Memory
|
|
121
|
+
|
|
122
|
+
A decoder does not know how many bytes it will produce until it has skipped
|
|
123
|
+
the spaces and line breaks, so fastpybase64 allocates the maximal length,
|
|
124
|
+
which counts them as data. When the result is shorter by at most 1/8, the
|
|
125
|
+
`bytes` object keeps its allocation: about 2.6% of it goes unused for MIME
|
|
126
|
+
input (a CRLF every 76 characters). Shrinking the allocation instead would
|
|
127
|
+
make large MIME inputs several times slower on Linux, because glibc would
|
|
128
|
+
then map fresh memory for every call. A larger difference (input that is
|
|
129
|
+
mostly spaces, for example) is given back.
|
|
130
|
+
|
|
131
|
+
`fastpybase64.set_exact_sizing(True)` sizes every decoded output exactly
|
|
132
|
+
for valid input, with an extra pass over the input, so that no memory goes
|
|
133
|
+
unused. Valid input contains only base64 alphabet characters, `=` padding,
|
|
134
|
+
and ASCII whitespace: any byte at or below 0x20, which covers a CRLF every
|
|
135
|
+
76 characters. A byte above 0x20 and outside the alphabet is counted in
|
|
136
|
+
that length and then ignored, so the allocation can be larger than the
|
|
137
|
+
output. Decoding is then slower: on an Intel Xeon Gold 6548N, 1 MiB decodes
|
|
138
|
+
at 14.0 GB/s instead of 19.1 GB/s, and 1 MiB of MIME data at 8.5 GB/s
|
|
139
|
+
instead of 12.0 GB/s. The setting applies to the whole process.
|
|
140
|
+
|
|
119
141
|
### Threads
|
|
120
142
|
|
|
121
143
|
The GIL is released while encoding or decoding 64 KiB or more, so several
|
|
@@ -128,7 +150,7 @@ change, so memory stays safe).
|
|
|
128
150
|
## Build and test
|
|
129
151
|
|
|
130
152
|
Python 3.10 or newer, and a C++17 compiler (clang, GCC or MSVC). The simdutf
|
|
131
|
-
9.
|
|
153
|
+
9.4.0 amalgamation is in `vendor/`. Only its base64 functions are compiled
|
|
132
154
|
(see `setup.py`).
|
|
133
155
|
|
|
134
156
|
pip:
|
|
@@ -46,6 +46,7 @@ base64.encodebytes(data) # MIME: lines of 76 characters
|
|
|
46
46
|
| `urlsafe_b64encode(s)`, `urlsafe_b64decode(s)` | alphabet with `-` and `_` |
|
|
47
47
|
| `encodebytes(s)`, `decodebytes(s)` | MIME style: a newline after every 76 characters |
|
|
48
48
|
| `get_implementation()` | the kernel simdutf selected for this processor (`icelake`, `haswell`, `arm64`, ...) |
|
|
49
|
+
| `set_exact_sizing(enabled)`, `get_exact_sizing()` | size decoded outputs exactly (off by default, see [Memory](#memory)) |
|
|
49
50
|
| `simdutf_version`, `__version__` | versions of simdutf and of fastpybase64 |
|
|
50
51
|
|
|
51
52
|
The functions accept what `base64` accepts: `bytes`, `bytearray`,
|
|
@@ -81,6 +82,27 @@ the same message, and the rare inputs where `base64` is lenient in odd ways
|
|
|
81
82
|
simdutf and then replacing the two characters; decoding with such an
|
|
82
83
|
alphabet is left to `base64`.
|
|
83
84
|
|
|
85
|
+
### Memory
|
|
86
|
+
|
|
87
|
+
A decoder does not know how many bytes it will produce until it has skipped
|
|
88
|
+
the spaces and line breaks, so fastpybase64 allocates the maximal length,
|
|
89
|
+
which counts them as data. When the result is shorter by at most 1/8, the
|
|
90
|
+
`bytes` object keeps its allocation: about 2.6% of it goes unused for MIME
|
|
91
|
+
input (a CRLF every 76 characters). Shrinking the allocation instead would
|
|
92
|
+
make large MIME inputs several times slower on Linux, because glibc would
|
|
93
|
+
then map fresh memory for every call. A larger difference (input that is
|
|
94
|
+
mostly spaces, for example) is given back.
|
|
95
|
+
|
|
96
|
+
`fastpybase64.set_exact_sizing(True)` sizes every decoded output exactly
|
|
97
|
+
for valid input, with an extra pass over the input, so that no memory goes
|
|
98
|
+
unused. Valid input contains only base64 alphabet characters, `=` padding,
|
|
99
|
+
and ASCII whitespace: any byte at or below 0x20, which covers a CRLF every
|
|
100
|
+
76 characters. A byte above 0x20 and outside the alphabet is counted in
|
|
101
|
+
that length and then ignored, so the allocation can be larger than the
|
|
102
|
+
output. Decoding is then slower: on an Intel Xeon Gold 6548N, 1 MiB decodes
|
|
103
|
+
at 14.0 GB/s instead of 19.1 GB/s, and 1 MiB of MIME data at 8.5 GB/s
|
|
104
|
+
instead of 12.0 GB/s. The setting applies to the whole process.
|
|
105
|
+
|
|
84
106
|
### Threads
|
|
85
107
|
|
|
86
108
|
The GIL is released while encoding or decoding 64 KiB or more, so several
|
|
@@ -93,7 +115,7 @@ change, so memory stays safe).
|
|
|
93
115
|
## Build and test
|
|
94
116
|
|
|
95
117
|
Python 3.10 or newer, and a C++17 compiler (clang, GCC or MSVC). The simdutf
|
|
96
|
-
9.
|
|
118
|
+
9.4.0 amalgamation is in `vendor/`. Only its base64 functions are compiled
|
|
97
119
|
(see `setup.py`).
|
|
98
120
|
|
|
99
121
|
pip:
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Compare fastpybase64 with the standard library and pybase64.
|
|
2
|
+
|
|
3
|
+
Usage: python bench.py [--sizes 16,1024,1048576] [--time 0.2] [--same-process]
|
|
4
|
+
[--exact-sizing]
|
|
5
|
+
|
|
6
|
+
Throughput is reported in GB/s of *binary* (decoded) data for both encoding
|
|
7
|
+
and decoding, so that the numbers of the two directions are comparable. For
|
|
8
|
+
each case we report the best of several repetitions.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import base64
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import platform
|
|
16
|
+
import random
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
import timeit
|
|
21
|
+
|
|
22
|
+
import fastpybase64
|
|
23
|
+
|
|
24
|
+
LIBRARIES = {"base64 (stdlib)": base64, "fastpybase64": fastpybase64}
|
|
25
|
+
try:
|
|
26
|
+
import pybase64
|
|
27
|
+
|
|
28
|
+
LIBRARIES["pybase64"] = pybase64
|
|
29
|
+
except ImportError:
|
|
30
|
+
pybase64 = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def mime_crlf(raw):
|
|
34
|
+
return base64.encodebytes(raw).replace(b"\n", b"\r\n")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# name -> (function name, keyword arguments, input preparation)
|
|
38
|
+
OPERATIONS = {
|
|
39
|
+
"b64encode": ("b64encode", {}, lambda raw: raw),
|
|
40
|
+
"b64decode": ("b64decode", {}, base64.b64encode),
|
|
41
|
+
"b64decode(validate=True)": ("b64decode", {"validate": True}, base64.b64encode),
|
|
42
|
+
"urlsafe_b64encode": ("urlsafe_b64encode", {}, lambda raw: raw),
|
|
43
|
+
"urlsafe_b64decode": ("urlsafe_b64decode", {}, base64.urlsafe_b64encode),
|
|
44
|
+
"encodebytes": ("encodebytes", {}, lambda raw: raw),
|
|
45
|
+
"decodebytes": ("decodebytes", {}, base64.encodebytes),
|
|
46
|
+
# MIME as found in email: lines of 76 characters ending with CRLF.
|
|
47
|
+
"b64decode(MIME CRLF)": ("b64decode", {}, mime_crlf),
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
DEFAULT_SIZES = [16, 64, 256, 1024, 4096, 65536, 1 << 20, 16 << 20]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def human_size(n):
|
|
54
|
+
for unit in ("B", "KiB", "MiB", "GiB"):
|
|
55
|
+
if n < 1024 or unit == "GiB":
|
|
56
|
+
return f"{n:g} {unit}"
|
|
57
|
+
n /= 1024
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def measure(func, arg, kwargs, min_time, repeat):
|
|
61
|
+
if kwargs:
|
|
62
|
+
def call():
|
|
63
|
+
func(arg, **kwargs)
|
|
64
|
+
else:
|
|
65
|
+
def call():
|
|
66
|
+
func(arg)
|
|
67
|
+
timer = timeit.Timer(call)
|
|
68
|
+
number, elapsed = timer.autorange()
|
|
69
|
+
number = max(1, int(number * min_time / max(elapsed, 1e-9)))
|
|
70
|
+
return min(timer.repeat(repeat=repeat, number=number)) / number
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def measure_libraries(op, arg, size, min_time, repeat):
|
|
74
|
+
"""Returns {library name: GB/s} for one operation on one input."""
|
|
75
|
+
func_name, kwargs, _ = OPERATIONS[op]
|
|
76
|
+
names = [n for n, m in LIBRARIES.items() if hasattr(m, func_name)]
|
|
77
|
+
speeds, outputs = {}, {}
|
|
78
|
+
# fastpybase64 goes first, and the results are checked afterwards: the
|
|
79
|
+
# other libraries may free large temporary buffers, which would change
|
|
80
|
+
# the allocator's state (see run_in_child).
|
|
81
|
+
for name in sorted(names, key=lambda n: n != "fastpybase64"):
|
|
82
|
+
func = getattr(LIBRARIES[name], func_name)
|
|
83
|
+
outputs[name] = func(arg, **kwargs)
|
|
84
|
+
seconds = measure(func, arg, kwargs, min_time, repeat)
|
|
85
|
+
speeds[name] = size / seconds / 1e9
|
|
86
|
+
expected = getattr(base64, func_name)(arg, **kwargs)
|
|
87
|
+
for name, output in outputs.items():
|
|
88
|
+
assert output == expected, (name, op, size)
|
|
89
|
+
return speeds
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def run_in_child(op, arg, size, args):
|
|
93
|
+
# Each measurement runs in a fresh process that reads its input from a
|
|
94
|
+
# file. The allocator remembers what was freed before: glibc raises its
|
|
95
|
+
# mmap threshold after freeing a large block, so building the input (or
|
|
96
|
+
# an earlier measurement) could hide the cost of allocating large
|
|
97
|
+
# outputs, a cost that a program decoding a file it just read does pay.
|
|
98
|
+
with tempfile.NamedTemporaryFile(suffix=".bench", delete=False) as f:
|
|
99
|
+
f.write(arg)
|
|
100
|
+
try:
|
|
101
|
+
out = subprocess.run(
|
|
102
|
+
[sys.executable, __file__, "--child", f.name, "--ops", op,
|
|
103
|
+
"--sizes", str(size), "--time", str(args.time),
|
|
104
|
+
"--repeat", str(args.repeat)]
|
|
105
|
+
+ (["--exact-sizing"] if args.exact_sizing else []),
|
|
106
|
+
check=True, stdout=subprocess.PIPE, text=True,
|
|
107
|
+
).stdout
|
|
108
|
+
finally:
|
|
109
|
+
os.unlink(f.name)
|
|
110
|
+
return json.loads(out)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def main():
|
|
114
|
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
115
|
+
parser.add_argument(
|
|
116
|
+
"--sizes",
|
|
117
|
+
default=",".join(map(str, DEFAULT_SIZES)),
|
|
118
|
+
help="comma-separated binary sizes in bytes",
|
|
119
|
+
)
|
|
120
|
+
parser.add_argument("--time", type=float, default=0.2, help="seconds per sample")
|
|
121
|
+
parser.add_argument("--repeat", type=int, default=5)
|
|
122
|
+
parser.add_argument(
|
|
123
|
+
"--ops", default=",".join(OPERATIONS), help="comma-separated operations"
|
|
124
|
+
)
|
|
125
|
+
parser.add_argument(
|
|
126
|
+
"--same-process",
|
|
127
|
+
action="store_true",
|
|
128
|
+
help="measure in this process (by default each measurement runs in a fresh one)",
|
|
129
|
+
)
|
|
130
|
+
parser.add_argument(
|
|
131
|
+
"--exact-sizing",
|
|
132
|
+
action="store_true",
|
|
133
|
+
help="call fastpybase64.set_exact_sizing(True) before measuring",
|
|
134
|
+
)
|
|
135
|
+
parser.add_argument("--child", metavar="FILE", help=argparse.SUPPRESS)
|
|
136
|
+
args = parser.parse_args()
|
|
137
|
+
sizes = [int(s) for s in args.sizes.split(",")]
|
|
138
|
+
ops = args.ops.split(",")
|
|
139
|
+
for op in ops:
|
|
140
|
+
if op not in OPERATIONS:
|
|
141
|
+
parser.error(f"unknown operation {op!r}; choose among {', '.join(OPERATIONS)}")
|
|
142
|
+
|
|
143
|
+
fastpybase64.set_exact_sizing(args.exact_sizing)
|
|
144
|
+
if args.child:
|
|
145
|
+
with open(args.child, "rb") as f:
|
|
146
|
+
arg = f.read()
|
|
147
|
+
speeds = measure_libraries(ops[0], arg, sizes[0], args.time, args.repeat)
|
|
148
|
+
print(json.dumps(speeds))
|
|
149
|
+
return
|
|
150
|
+
print_header()
|
|
151
|
+
run(ops, sizes, args)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def print_header():
|
|
155
|
+
print(f"Python {sys.version.split()[0]} on {platform.machine()} ({platform.platform()})")
|
|
156
|
+
print(
|
|
157
|
+
f"fastpybase64 {fastpybase64.__version__} "
|
|
158
|
+
f"(simdutf {fastpybase64.simdutf_version}, kernel: "
|
|
159
|
+
f"{fastpybase64.get_implementation()}"
|
|
160
|
+
f"{', exact sizing' if fastpybase64.get_exact_sizing() else ''})"
|
|
161
|
+
)
|
|
162
|
+
if pybase64 is not None:
|
|
163
|
+
print(f"pybase64 {pybase64.get_version()}")
|
|
164
|
+
print()
|
|
165
|
+
print("Throughput in GB/s of binary data (higher is better).")
|
|
166
|
+
print(flush=True)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def run(ops, sizes, args):
|
|
170
|
+
rng = random.Random(42)
|
|
171
|
+
for op in ops:
|
|
172
|
+
func_name, _, prepare = OPERATIONS[op]
|
|
173
|
+
print(f"### {op}")
|
|
174
|
+
print()
|
|
175
|
+
names = [n for n, m in LIBRARIES.items() if hasattr(m, func_name)]
|
|
176
|
+
header = ["size"] + names + ["speedup vs stdlib", "speedup vs pybase64"]
|
|
177
|
+
print("| " + " | ".join(header) + " |")
|
|
178
|
+
print("|" + "|".join("---:" for _ in header) + "|", flush=True)
|
|
179
|
+
for size in sizes:
|
|
180
|
+
arg = prepare(rng.randbytes(size))
|
|
181
|
+
if args.same_process:
|
|
182
|
+
speeds = measure_libraries(op, arg, size, args.time, args.repeat)
|
|
183
|
+
else:
|
|
184
|
+
speeds = run_in_child(op, arg, size, args)
|
|
185
|
+
row = [human_size(size)] + [f"{speeds[n]:.2f}" for n in names]
|
|
186
|
+
row.append(f"{speeds['fastpybase64'] / speeds['base64 (stdlib)']:.1f}x")
|
|
187
|
+
if "pybase64" in speeds:
|
|
188
|
+
row.append(f"{speeds['fastpybase64'] / speeds['pybase64']:.2f}x")
|
|
189
|
+
else:
|
|
190
|
+
row.append("n/a")
|
|
191
|
+
print("| " + " | ".join(row) + " |", flush=True)
|
|
192
|
+
print()
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
if __name__ == "__main__":
|
|
196
|
+
main()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fastpybase64
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Fast Base64 encoding and decoding for Python, built on simdutf
|
|
5
5
|
Author-email: Daniel Lemire <daniel@lemire.me>
|
|
6
6
|
License-Expression: Apache-2.0 OR MIT
|
|
@@ -81,6 +81,7 @@ base64.encodebytes(data) # MIME: lines of 76 characters
|
|
|
81
81
|
| `urlsafe_b64encode(s)`, `urlsafe_b64decode(s)` | alphabet with `-` and `_` |
|
|
82
82
|
| `encodebytes(s)`, `decodebytes(s)` | MIME style: a newline after every 76 characters |
|
|
83
83
|
| `get_implementation()` | the kernel simdutf selected for this processor (`icelake`, `haswell`, `arm64`, ...) |
|
|
84
|
+
| `set_exact_sizing(enabled)`, `get_exact_sizing()` | size decoded outputs exactly (off by default, see [Memory](#memory)) |
|
|
84
85
|
| `simdutf_version`, `__version__` | versions of simdutf and of fastpybase64 |
|
|
85
86
|
|
|
86
87
|
The functions accept what `base64` accepts: `bytes`, `bytearray`,
|
|
@@ -116,6 +117,27 @@ the same message, and the rare inputs where `base64` is lenient in odd ways
|
|
|
116
117
|
simdutf and then replacing the two characters; decoding with such an
|
|
117
118
|
alphabet is left to `base64`.
|
|
118
119
|
|
|
120
|
+
### Memory
|
|
121
|
+
|
|
122
|
+
A decoder does not know how many bytes it will produce until it has skipped
|
|
123
|
+
the spaces and line breaks, so fastpybase64 allocates the maximal length,
|
|
124
|
+
which counts them as data. When the result is shorter by at most 1/8, the
|
|
125
|
+
`bytes` object keeps its allocation: about 2.6% of it goes unused for MIME
|
|
126
|
+
input (a CRLF every 76 characters). Shrinking the allocation instead would
|
|
127
|
+
make large MIME inputs several times slower on Linux, because glibc would
|
|
128
|
+
then map fresh memory for every call. A larger difference (input that is
|
|
129
|
+
mostly spaces, for example) is given back.
|
|
130
|
+
|
|
131
|
+
`fastpybase64.set_exact_sizing(True)` sizes every decoded output exactly
|
|
132
|
+
for valid input, with an extra pass over the input, so that no memory goes
|
|
133
|
+
unused. Valid input contains only base64 alphabet characters, `=` padding,
|
|
134
|
+
and ASCII whitespace: any byte at or below 0x20, which covers a CRLF every
|
|
135
|
+
76 characters. A byte above 0x20 and outside the alphabet is counted in
|
|
136
|
+
that length and then ignored, so the allocation can be larger than the
|
|
137
|
+
output. Decoding is then slower: on an Intel Xeon Gold 6548N, 1 MiB decodes
|
|
138
|
+
at 14.0 GB/s instead of 19.1 GB/s, and 1 MiB of MIME data at 8.5 GB/s
|
|
139
|
+
instead of 12.0 GB/s. The setting applies to the whole process.
|
|
140
|
+
|
|
119
141
|
### Threads
|
|
120
142
|
|
|
121
143
|
The GIL is released while encoding or decoding 64 KiB or more, so several
|
|
@@ -128,7 +150,7 @@ change, so memory stays safe).
|
|
|
128
150
|
## Build and test
|
|
129
151
|
|
|
130
152
|
Python 3.10 or newer, and a C++17 compiler (clang, GCC or MSVC). The simdutf
|
|
131
|
-
9.
|
|
153
|
+
9.4.0 amalgamation is in `vendor/`. Only its base64 functions are compiled
|
|
132
154
|
(see `setup.py`).
|
|
133
155
|
|
|
134
156
|
pip:
|
|
@@ -10,9 +10,10 @@
|
|
|
10
10
|
#define PY_SSIZE_T_CLEAN
|
|
11
11
|
#include <Python.h>
|
|
12
12
|
|
|
13
|
+
#include <atomic>
|
|
13
14
|
#include <cstring>
|
|
14
15
|
|
|
15
|
-
#define FASTPYBASE64_VERSION "0.
|
|
16
|
+
#define FASTPYBASE64_VERSION "0.2.0"
|
|
16
17
|
|
|
17
18
|
// Unity build: compiling simdutf in this translation unit keeps the build
|
|
18
19
|
// simple and lets the compiler inline across the boundary.
|
|
@@ -23,6 +24,10 @@ namespace {
|
|
|
23
24
|
// Release the GIL when processing at least this many bytes.
|
|
24
25
|
constexpr Py_ssize_t kReleaseGilThreshold = 64 * 1024;
|
|
25
26
|
|
|
27
|
+
// set_exact_sizing(): size decoded outputs exactly (one more pass over the
|
|
28
|
+
// input) instead of allocating the maximal length. Process-wide.
|
|
29
|
+
std::atomic<bool> exact_sizing{false};
|
|
30
|
+
|
|
26
31
|
// stdlib.encodebytes emits lines of at most 76 characters.
|
|
27
32
|
constexpr size_t kMimeLineLength = 76;
|
|
28
33
|
|
|
@@ -318,15 +323,34 @@ bool garbage_result_ok(const char *in, size_t length, size_t decoded,
|
|
|
318
323
|
return true;
|
|
319
324
|
}
|
|
320
325
|
|
|
326
|
+
// Decodes into out, which holds capacity bytes. On success, r.count is the
|
|
327
|
+
// number of bytes written. A capacity below the maximal binary length (exact
|
|
328
|
+
// sizing) requires the bounds-checked decoder.
|
|
329
|
+
simdutf::result decode_into(const InputView &input, char *out,
|
|
330
|
+
size_t capacity, size_t max_length,
|
|
331
|
+
simdutf::base64_options options) {
|
|
332
|
+
if (capacity >= max_length) {
|
|
333
|
+
return simdutf::base64_to_binary(input.data, input.length, out, options);
|
|
334
|
+
}
|
|
335
|
+
size_t written = capacity;
|
|
336
|
+
simdutf::result r = simdutf::base64_to_binary_safe(
|
|
337
|
+
input.data, input.length, out, written, options);
|
|
338
|
+
if (r.error == simdutf::error_code::SUCCESS) {
|
|
339
|
+
r.count = written;
|
|
340
|
+
}
|
|
341
|
+
return r;
|
|
342
|
+
}
|
|
343
|
+
|
|
321
344
|
simdutf::result run_decoder(const InputView &input, char *out,
|
|
345
|
+
size_t capacity, size_t max_length,
|
|
322
346
|
simdutf::base64_options options) {
|
|
323
347
|
simdutf::result r;
|
|
324
348
|
if (Py_ssize_t(input.length) >= kReleaseGilThreshold) {
|
|
325
349
|
Py_BEGIN_ALLOW_THREADS
|
|
326
|
-
r =
|
|
350
|
+
r = decode_into(input, out, capacity, max_length, options);
|
|
327
351
|
Py_END_ALLOW_THREADS
|
|
328
352
|
} else {
|
|
329
|
-
r =
|
|
353
|
+
r = decode_into(input, out, capacity, max_length, options);
|
|
330
354
|
}
|
|
331
355
|
return r;
|
|
332
356
|
}
|
|
@@ -340,7 +364,14 @@ PyObject *try_decode(const InputView &input, bool url, bool validate) {
|
|
|
340
364
|
}
|
|
341
365
|
size_t max_length =
|
|
342
366
|
simdutf::maximal_binary_length_from_base64(input.data, input.length);
|
|
343
|
-
|
|
367
|
+
// With exact sizing, binary_length_from_base64 skips spaces and line
|
|
368
|
+
// breaks: the result is exact for valid input and an upper bound for any
|
|
369
|
+
// input (characters outside the alphabet are counted).
|
|
370
|
+
size_t capacity = max_length;
|
|
371
|
+
if (exact_sizing.load(std::memory_order_relaxed)) {
|
|
372
|
+
capacity = simdutf::binary_length_from_base64(input.data, input.length);
|
|
373
|
+
}
|
|
374
|
+
PyObject *result = PyBytes_FromStringAndSize(nullptr, Py_ssize_t(capacity));
|
|
344
375
|
if (result == nullptr) {
|
|
345
376
|
return nullptr;
|
|
346
377
|
}
|
|
@@ -348,7 +379,7 @@ PyObject *try_decode(const InputView &input, bool url, bool validate) {
|
|
|
348
379
|
|
|
349
380
|
simdutf::base64_options options =
|
|
350
381
|
url ? simdutf::base64_default_or_url : simdutf::base64_default;
|
|
351
|
-
simdutf::result r = run_decoder(input, out, options);
|
|
382
|
+
simdutf::result r = run_decoder(input, out, capacity, max_length, options);
|
|
352
383
|
bool ok;
|
|
353
384
|
if (r.error == simdutf::error_code::SUCCESS) {
|
|
354
385
|
ok = validate ? strict_result_ok(input.data, input.length, r.count)
|
|
@@ -357,7 +388,7 @@ PyObject *try_decode(const InputView &input, bool url, bool validate) {
|
|
|
357
388
|
// The input may contain characters that the standard library discards.
|
|
358
389
|
options = url ? simdutf::base64_default_or_url_accept_garbage
|
|
359
390
|
: simdutf::base64_default_accept_garbage;
|
|
360
|
-
r = run_decoder(input, out, options);
|
|
391
|
+
r = run_decoder(input, out, capacity, max_length, options);
|
|
361
392
|
ok = r.error == simdutf::error_code::SUCCESS &&
|
|
362
393
|
garbage_result_ok(input.data, input.length, r.count, url);
|
|
363
394
|
} else {
|
|
@@ -367,8 +398,11 @@ PyObject *try_decode(const InputView &input, bool url, bool validate) {
|
|
|
367
398
|
Py_DECREF(result);
|
|
368
399
|
return nullptr;
|
|
369
400
|
}
|
|
370
|
-
if (r.count !=
|
|
371
|
-
if (
|
|
401
|
+
if (r.count != capacity) {
|
|
402
|
+
if (capacity - r.count <= capacity / 8) {
|
|
403
|
+
Py_SET_SIZE(result, Py_ssize_t(r.count));
|
|
404
|
+
out[r.count] = '\0';
|
|
405
|
+
} else if (_PyBytes_Resize(&result, Py_ssize_t(r.count)) != 0) {
|
|
372
406
|
return nullptr;
|
|
373
407
|
}
|
|
374
408
|
}
|
|
@@ -554,6 +588,19 @@ PyObject *get_implementation(PyObject *, PyObject *) {
|
|
|
554
588
|
return PyUnicode_FromStringAndSize(name.data(), Py_ssize_t(name.size()));
|
|
555
589
|
}
|
|
556
590
|
|
|
591
|
+
PyObject *set_exact_sizing(PyObject *, PyObject *arg) {
|
|
592
|
+
int enabled = PyObject_IsTrue(arg);
|
|
593
|
+
if (enabled < 0) {
|
|
594
|
+
return nullptr;
|
|
595
|
+
}
|
|
596
|
+
exact_sizing.store(enabled != 0, std::memory_order_relaxed);
|
|
597
|
+
Py_RETURN_NONE;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
PyObject *get_exact_sizing(PyObject *, PyObject *) {
|
|
601
|
+
return PyBool_FromLong(exact_sizing.load(std::memory_order_relaxed));
|
|
602
|
+
}
|
|
603
|
+
|
|
557
604
|
#define FASTCALL_KW(f) reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)(void)>(f))
|
|
558
605
|
|
|
559
606
|
PyMethodDef methods[] = {
|
|
@@ -589,6 +636,18 @@ PyMethodDef methods[] = {
|
|
|
589
636
|
{"get_implementation", get_implementation, METH_NOARGS,
|
|
590
637
|
"get_implementation()\n--\n\n"
|
|
591
638
|
"Return the name of the SIMD kernel selected by simdutf at runtime."},
|
|
639
|
+
{"set_exact_sizing", set_exact_sizing, METH_O,
|
|
640
|
+
"set_exact_sizing(enabled, /)\n--\n\n"
|
|
641
|
+
"Size decoded outputs exactly (default: False).\n\n"
|
|
642
|
+
"By default, the decoders allocate the maximal output length, which\n"
|
|
643
|
+
"counts spaces and line breaks as data, and may leave up to 1/8 of the\n"
|
|
644
|
+
"allocation unused (about 2.6% for MIME input). With exact sizing, an\n"
|
|
645
|
+
"extra pass over the input computes the exact length: no memory is\n"
|
|
646
|
+
"wasted, but decoding is slower. The setting applies to the whole\n"
|
|
647
|
+
"process."},
|
|
648
|
+
{"get_exact_sizing", get_exact_sizing, METH_NOARGS,
|
|
649
|
+
"get_exact_sizing()\n--\n\n"
|
|
650
|
+
"Return whether decoded outputs are sized exactly."},
|
|
592
651
|
{nullptr, nullptr, 0, nullptr},
|
|
593
652
|
};
|
|
594
653
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
import fastpybase64
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@pytest.fixture(autouse=True, params=[False, True], ids=["default", "exact-sizing"])
|
|
7
|
+
def sizing(request):
|
|
8
|
+
"""Runs every test with and without exact output sizing."""
|
|
9
|
+
previous = fastpybase64.get_exact_sizing()
|
|
10
|
+
fastpybase64.set_exact_sizing(request.param)
|
|
11
|
+
yield request.param
|
|
12
|
+
fastpybase64.set_exact_sizing(previous)
|
|
@@ -258,3 +258,25 @@ def test_large_with_noise():
|
|
|
258
258
|
check_decoders(encoded)
|
|
259
259
|
check_decoders(encoded.replace(b"\n", b"!\n"))
|
|
260
260
|
check_decoders(encoded + b"YQ==")
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def test_exact_sizing_setting(sizing):
|
|
264
|
+
assert fastpybase64.get_exact_sizing() is sizing
|
|
265
|
+
fastpybase64.set_exact_sizing(0)
|
|
266
|
+
assert fastpybase64.get_exact_sizing() is False
|
|
267
|
+
fastpybase64.set_exact_sizing([1])
|
|
268
|
+
assert fastpybase64.get_exact_sizing() is True
|
|
269
|
+
with pytest.raises(TypeError):
|
|
270
|
+
fastpybase64.set_exact_sizing()
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
@pytest.mark.parametrize("length", [0, 1, 2, 3, 57, 1000, 1 << 20])
|
|
274
|
+
def test_mime_crlf(length):
|
|
275
|
+
data = random.Random(length).randbytes(length)
|
|
276
|
+
encoded = base64.encodebytes(data).replace(b"\n", b"\r\n")
|
|
277
|
+
check_decoders(encoded)
|
|
278
|
+
check_decoders(b" \r\n\t" + encoded + b"\r\n \r\n")
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def test_only_whitespace():
|
|
282
|
+
check_decoders(b" \r\n" * 1000)
|