fastsimdjson 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastsimdjson-0.1.0/MANIFEST.in +2 -0
- fastsimdjson-0.1.0/PKG-INFO +222 -0
- fastsimdjson-0.1.0/README.md +194 -0
- fastsimdjson-0.1.0/pyproject.toml +55 -0
- fastsimdjson-0.1.0/setup.cfg +4 -0
- fastsimdjson-0.1.0/setup.py +28 -0
- fastsimdjson-0.1.0/src/fastsimdjson.cpp +736 -0
- fastsimdjson-0.1.0/src/fastsimdjson.egg-info/PKG-INFO +222 -0
- fastsimdjson-0.1.0/src/fastsimdjson.egg-info/SOURCES.txt +16 -0
- fastsimdjson-0.1.0/src/fastsimdjson.egg-info/dependency_links.txt +1 -0
- fastsimdjson-0.1.0/src/fastsimdjson.egg-info/requires.txt +6 -0
- fastsimdjson-0.1.0/src/fastsimdjson.egg-info/top_level.txt +1 -0
- fastsimdjson-0.1.0/tests/test_loads.py +349 -0
- fastsimdjson-0.1.0/vendor/simdjson.cpp +80877 -0
- fastsimdjson-0.1.0/vendor/simdjson.h +242184 -0
- fastsimdjson-0.1.0/vendor/simdutf.cpp +69515 -0
- fastsimdjson-0.1.0/vendor/simdutf.h +14536 -0
- fastsimdjson-0.1.0/vendor/simdutf_c.h +382 -0
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fastsimdjson
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fast JSON parsing for Python, built on simdjson
|
|
5
|
+
Author-email: Daniel Lemire <daniel@lemire.me>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/simdjson/fastpysimdjson
|
|
8
|
+
Project-URL: Repository, https://github.com/simdjson/fastpysimdjson
|
|
9
|
+
Project-URL: Issues, https://github.com/simdjson/fastpysimdjson/issues
|
|
10
|
+
Keywords: json,simdjson,parser,simd,performance
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: C++
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
Provides-Extra: test
|
|
25
|
+
Requires-Dist: pytest; extra == "test"
|
|
26
|
+
Provides-Extra: bench
|
|
27
|
+
Requires-Dist: orjson; extra == "bench"
|
|
28
|
+
|
|
29
|
+
# fastsimdjson
|
|
30
|
+
|
|
31
|
+
A Python binding for [simdjson](https://github.com/simdjson/simdjson) that
|
|
32
|
+
parses JSON into native Python objects (`dict`, `list`, `str`, `int`, `float`,
|
|
33
|
+
`bool`, `None`). It is intended as a faster drop-in for `orjson.loads` /
|
|
34
|
+
`json.loads`.
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
import fastsimdjson
|
|
38
|
+
fastsimdjson.loads(b'{"a": [1, 2.5, "x", true, null]}')
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
`loads` accepts `bytes`, `bytearray`, `memoryview` and `str`. Invalid input
|
|
42
|
+
raises `fastsimdjson.JSONDecodeError`, a subclass of `json.JSONDecodeError`.
|
|
43
|
+
When simdjson rejects a document, the same input is parsed with `json.loads`.
|
|
44
|
+
If that succeeds, `loads` returns its value. If it raises `JSONDecodeError`,
|
|
45
|
+
the exception is re-raised with Python's message and byte position. Any other
|
|
46
|
+
exception from `json.loads` propagates. `release()` frees the simdjson parser
|
|
47
|
+
and the string caches kept by the calling thread.
|
|
48
|
+
|
|
49
|
+
## How it works
|
|
50
|
+
|
|
51
|
+
1. simdjson's DOM parser (with runtime CPU dispatch: AVX-512, AVX2, SSE4.2, ...)
|
|
52
|
+
validates the document and builds its tape. Each thread keeps its own
|
|
53
|
+
parser and reuses it across calls; `release()` deletes that parser and
|
|
54
|
+
drops the thread's cached strings. The document is parsed in place. When
|
|
55
|
+
the 64 bytes simdjson would read past the end cross a page boundary, the
|
|
56
|
+
unpadded DOM parser is used instead of copying the buffer.
|
|
57
|
+
2. A tape walker creates the Python objects directly:
|
|
58
|
+
* lists are allocated at their final size (simdjson records element counts);
|
|
59
|
+
scalars are handled inline in the array/object loops;
|
|
60
|
+
* dicts are presized and filled with `_PyDict_SetItem_KnownHash`;
|
|
61
|
+
* object keys go through a direct-mapped cache (ASCII keys of up to 64
|
|
62
|
+
bytes), so repeated keys reuse one `str` object whose hash is already
|
|
63
|
+
computed. Short ASCII string values (up to 16 bytes) have their own cache.
|
|
64
|
+
Both caches belong to the calling thread;
|
|
65
|
+
* strings are built with `PyUnicode_New` plus a copy. Non-ASCII UTF-8,
|
|
66
|
+
which simdjson has already validated, is transcoded without
|
|
67
|
+
revalidation: scalar code for short strings, simdutf for long ones;
|
|
68
|
+
* on builds that use the GIL, the cyclic GC is paused while objects are built.
|
|
69
|
+
3. Integers that do not fit in 64 bits stay on the tape as digit strings and
|
|
70
|
+
become exact Python ints (orjson turns them into floats).
|
|
71
|
+
|
|
72
|
+
`NaN`, `Infinity`, and `-Infinity` parse as floats, matching `json.loads`.
|
|
73
|
+
simdjson is built with `SIMDJSON_ENABLE_NAN_INF`, so any capitalization of
|
|
74
|
+
`nan`, `inf`, and `infinity` is also accepted (Python's parser only allows
|
|
75
|
+
the three spellings above). A number that overflows a double becomes
|
|
76
|
+
`inf`, via `json.loads`, because simdjson still rejects it. An unpaired
|
|
77
|
+
surrogate escape (`"\ud800"`) is also rejected by simdjson; `json.loads`
|
|
78
|
+
accepts it, so `loads` returns that string. A leading UTF-8 BOM is accepted.
|
|
79
|
+
|
|
80
|
+
## Build and test
|
|
81
|
+
|
|
82
|
+
Python 3.10 or newer, and a C++17 compiler (clang or GCC). The simdjson 5.0.1
|
|
83
|
+
and simdutf 9.2.1 amalgamations are already in `vendor/`.
|
|
84
|
+
|
|
85
|
+
pip:
|
|
86
|
+
|
|
87
|
+
```sh
|
|
88
|
+
python3 -m venv .venv
|
|
89
|
+
. .venv/bin/activate
|
|
90
|
+
python -m pip install -U pip setuptools
|
|
91
|
+
python -m pip install -e ".[test]"
|
|
92
|
+
pytest tests
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
uv:
|
|
96
|
+
|
|
97
|
+
```sh
|
|
98
|
+
uv venv
|
|
99
|
+
. .venv/bin/activate
|
|
100
|
+
uv pip install -e ".[test]"
|
|
101
|
+
pytest tests
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Either one builds the extension and makes `import fastsimdjson` work in the
|
|
105
|
+
virtualenv. uv uses the `setuptools` build requirement from `pyproject.toml`,
|
|
106
|
+
so it does not need a separate setuptools install for this path.
|
|
107
|
+
|
|
108
|
+
To compile the extension in the tree instead, install setuptools and pytest
|
|
109
|
+
into the same virtualenv, then:
|
|
110
|
+
|
|
111
|
+
```sh
|
|
112
|
+
python -m pip install setuptools pytest
|
|
113
|
+
python setup.py build_ext --inplace
|
|
114
|
+
PYTHONPATH=src python -m pytest tests
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
```sh
|
|
118
|
+
uv pip install setuptools pytest
|
|
119
|
+
python setup.py build_ext --inplace
|
|
120
|
+
PYTHONPATH=src python -m pytest tests
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Recent setuptools copies the `.so` next to `src/fastsimdjson.cpp`, which is
|
|
124
|
+
why `PYTHONPATH=src` is required for the in-place build.
|
|
125
|
+
|
|
126
|
+
`tests/test_loads.py` compares `loads` with `json.loads` on types and key
|
|
127
|
+
order. It covers scalars, integers past 64 bits, UTF-8 strings at every
|
|
128
|
+
length from 0 to 199, the key cache, random documents, rejected input, deep
|
|
129
|
+
nesting, padding at a page boundary, a saturated array count, reference
|
|
130
|
+
counts, and release of a parser that has grown past 64 MB. The corpus test
|
|
131
|
+
is skipped until simdjson-data is checked out beside the project:
|
|
132
|
+
|
|
133
|
+
```sh
|
|
134
|
+
git clone --depth 1 https://github.com/simdjson/simdjson-data.git
|
|
135
|
+
pytest tests
|
|
136
|
+
# or: JSONDIR=/path/to/jsonexamples pytest tests
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The suite builds an ~80 MB document and a list of 16,777,221 integers, so
|
|
140
|
+
give it some RAM.
|
|
141
|
+
|
|
142
|
+
To time `loads` against `json.loads` and orjson on those files:
|
|
143
|
+
|
|
144
|
+
```sh
|
|
145
|
+
python -m pip install -e ".[bench]"
|
|
146
|
+
python bench.py
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
```sh
|
|
150
|
+
uv pip install -e ".[bench]"
|
|
151
|
+
python bench.py
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
## Benchmarks
|
|
155
|
+
|
|
156
|
+
Full materialization of `simdjson-data` on an Intel Xeon Gold 6548N, pinned
|
|
157
|
+
to one core. Python 3.12.13, GCC, `json.loads`, orjson 3.12.0, yyjson 4.0.6,
|
|
158
|
+
cysimdjson 26.27 (simdjson 3.8.0, AVX-512), pysimdjson 7.0.2 (icelake,
|
|
159
|
+
AVX-512). Times are microseconds, the best of several runs. Speedup is
|
|
160
|
+
orjson time / fastsimdjson time.
|
|
161
|
+
|
|
162
|
+
cysimdjson is `JSONParser.parse(data).export()` and pysimdjson is
|
|
163
|
+
`Parser.parse(data, recursive=True)`. `yyjson.loads` is `Document.as_obj`.
|
|
164
|
+
fastsimdjson, `json.loads`, cysimdjson, and pysimdjson matched orjson on
|
|
165
|
+
every file, including types and key order.
|
|
166
|
+
|
|
167
|
+
Geomean speedup of fastsimdjson: 3.38× over `json.loads`, 1.27× over orjson,
|
|
168
|
+
1.39× over yyjson, 1.83× over cysimdjson, and 1.79× over pysimdjson. It was
|
|
169
|
+
the fastest on 21 files. `numbers.json`, a flat array of floats, is the loss
|
|
170
|
+
against orjson (0.95×): the time is simdjson's float parser plus allocating
|
|
171
|
+
Python floats. `json.loads` is slowest on every file; the gap is largest on
|
|
172
|
+
`canada.json` (about 5.8×).
|
|
173
|
+
|
|
174
|
+
yyjson 4.0.6 returns non-ASCII strings as the raw UTF-8 bytes stored in a
|
|
175
|
+
Latin-1 `str`. Twelve files therefore do not match orjson (the twitter
|
|
176
|
+
files, `citm_catalog`, `gsoc-2018`, `github_events`, `random`, `repeat`,
|
|
177
|
+
`semanticscholar-corpus`, and `update-center`). Those yyjson times skip a
|
|
178
|
+
real UTF-8 decode. fastsimdjson is still ahead on the ten files where the
|
|
179
|
+
values match.
|
|
180
|
+
|
|
181
|
+
| file | KB | json | orjson | fast | yyjson | cys | psy | speedup |
|
|
182
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
183
|
+
| apache_builds.json | 124 | 383.2 | 180.7 | 153.9 | 205.9 | 263.1 | 264.8 | 1.17 |
|
|
184
|
+
| canada.json | 2198 | 30404.4 | 6518.2 | 5248.1 | 5552.2 | 7894.4 | 7348.4 | 1.24 |
|
|
185
|
+
| citm_catalog.json | 1687 | 7295.1 | 2934.6 | 1775.7 | 2554.0 | 4321.0 | 4206.3 | 1.65 |
|
|
186
|
+
| github_events.json | 64 | 216.1 | 68.5 | 55.6 | 82.8 | 98.5 | 97.7 | 1.23 |
|
|
187
|
+
| google_maps_api_compact_response.json | 12 | 80.3 | 39.7 | 33.6 | 47.3 | 55.6 | 56.5 | 1.18 |
|
|
188
|
+
| google_maps_api_response.json | 25 | 93.6 | 43.4 | 34.8 | 52.1 | 56.9 | 57.7 | 1.25 |
|
|
189
|
+
| gsoc-2018.json | 3250 | 7039.9 | 3889.8 | 2280.5 | 2968.0 | 3569.8 | 3611.0 | 1.71 |
|
|
190
|
+
| instruments.json | 215 | 848.4 | 319.3 | 239.1 | 405.1 | 478.6 | 500.7 | 1.34 |
|
|
191
|
+
| marine_ik.json | 2914 | 23662.6 | 9301.2 | 7272.7 | 8548.5 | 12315.3 | 11046.9 | 1.28 |
|
|
192
|
+
| mesh.json | 707 | 5093.0 | 1712.2 | 1493.1 | 1685.6 | 2269.3 | 2074.4 | 1.15 |
|
|
193
|
+
| mesh.pretty.json | 1540 | 8382.9 | 2435.5 | 1674.3 | 2185.0 | 2548.5 | 2309.9 | 1.45 |
|
|
194
|
+
| numbers.json | 147 | 914.3 | 221.4 | 232.3 | 260.4 | 299.2 | 276.2 | 0.95 |
|
|
195
|
+
| random.json | 499 | 3107.1 | 1522.5 | 1292.4 | 1632.5 | 2571.5 | 2390.8 | 1.18 |
|
|
196
|
+
| repeat.json | 11 | 43.9 | 15.3 | 13.0 | 16.7 | 30.5 | 24.1 | 1.18 |
|
|
197
|
+
| semanticscholar-corpus.json | 8392 | 45587.1 | 20634.5 | 14113.1 | 18832.2 | 28079.0 | 26477.3 | 1.46 |
|
|
198
|
+
| tree-pretty.json | 34 | 117.5 | 43.5 | 36.3 | 62.5 | 63.5 | 65.4 | 1.20 |
|
|
199
|
+
| twitter.json | 617 | 2798.4 | 1017.4 | 715.4 | 1168.0 | 1771.2 | 1825.1 | 1.42 |
|
|
200
|
+
| twitter_api_compact_response.json | 10 | 45.0 | 14.8 | 12.1 | 19.3 | 22.6 | 22.7 | 1.22 |
|
|
201
|
+
| twitter_api_response.json | 15 | 60.3 | 17.4 | 13.9 | 23.4 | 26.2 | 26.4 | 1.25 |
|
|
202
|
+
| twitter_timeline.json | 41 | 176.7 | 61.2 | 52.6 | 82.8 | 110.9 | 114.6 | 1.16 |
|
|
203
|
+
| twitterescaped.json | 549 | 2308.5 | 1021.7 | 801.4 | 1184.0 | 1961.0 | 2030.3 | 1.27 |
|
|
204
|
+
| update-center.json | 521 | 2675.8 | 1363.6 | 1119.1 | 1509.2 | 1982.1 | 2057.0 | 1.22 |
|
|
205
|
+
|
|
206
|
+
## Limitations
|
|
207
|
+
|
|
208
|
+
* Only `loads` is implemented; there is no `dumps`.
|
|
209
|
+
* The simdjson parser and the key and string caches are thread-local.
|
|
210
|
+
`release()` frees the parser and the cached strings retained by the calling
|
|
211
|
+
thread. A parser that grows past 64 MB is freed on its own at the end of
|
|
212
|
+
that call; its caches stay. A thread that exits without `release()` leaves
|
|
213
|
+
its cached strings behind. The module is marked free-threading compatible
|
|
214
|
+
(`Py_MOD_GIL_NOT_USED` on Python 3.13 and newer), so importing it on a
|
|
215
|
+
free-threaded build does not re-enable the GIL. Subinterpreters are not
|
|
216
|
+
supported. On a free-threaded build, `bytearray` and `memoryview` inputs
|
|
217
|
+
are copied before parsing.
|
|
218
|
+
* A document simdjson rejects is reparsed with `json.loads`. `loads` returns
|
|
219
|
+
that value when `json.loads` accepts it, which is how overflow to infinity
|
|
220
|
+
is handled. An exception is raised only when `json.loads` also fails.
|
|
221
|
+
`JSONDecodeError` is re-raised as `fastsimdjson.JSONDecodeError` with the
|
|
222
|
+
same message, document, and position. Any other exception propagates.
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# fastsimdjson
|
|
2
|
+
|
|
3
|
+
A Python binding for [simdjson](https://github.com/simdjson/simdjson) that
|
|
4
|
+
parses JSON into native Python objects (`dict`, `list`, `str`, `int`, `float`,
|
|
5
|
+
`bool`, `None`). It is intended as a faster drop-in for `orjson.loads` /
|
|
6
|
+
`json.loads`.
|
|
7
|
+
|
|
8
|
+
```python
|
|
9
|
+
import fastsimdjson
|
|
10
|
+
fastsimdjson.loads(b'{"a": [1, 2.5, "x", true, null]}')
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
`loads` accepts `bytes`, `bytearray`, `memoryview` and `str`. Invalid input
|
|
14
|
+
raises `fastsimdjson.JSONDecodeError`, a subclass of `json.JSONDecodeError`.
|
|
15
|
+
When simdjson rejects a document, the same input is parsed with `json.loads`.
|
|
16
|
+
If that succeeds, `loads` returns its value. If it raises `JSONDecodeError`,
|
|
17
|
+
the exception is re-raised with Python's message and byte position. Any other
|
|
18
|
+
exception from `json.loads` propagates. `release()` frees the simdjson parser
|
|
19
|
+
and the string caches kept by the calling thread.
|
|
20
|
+
|
|
21
|
+
## How it works
|
|
22
|
+
|
|
23
|
+
1. simdjson's DOM parser (with runtime CPU dispatch: AVX-512, AVX2, SSE4.2, ...)
|
|
24
|
+
validates the document and builds its tape. Each thread keeps its own
|
|
25
|
+
parser and reuses it across calls; `release()` deletes that parser and
|
|
26
|
+
drops the thread's cached strings. The document is parsed in place. When
|
|
27
|
+
the 64 bytes simdjson would read past the end cross a page boundary, the
|
|
28
|
+
unpadded DOM parser is used instead of copying the buffer.
|
|
29
|
+
2. A tape walker creates the Python objects directly:
|
|
30
|
+
* lists are allocated at their final size (simdjson records element counts);
|
|
31
|
+
scalars are handled inline in the array/object loops;
|
|
32
|
+
* dicts are presized and filled with `_PyDict_SetItem_KnownHash`;
|
|
33
|
+
* object keys go through a direct-mapped cache (ASCII keys of up to 64
|
|
34
|
+
bytes), so repeated keys reuse one `str` object whose hash is already
|
|
35
|
+
computed. Short ASCII string values (up to 16 bytes) have their own cache.
|
|
36
|
+
Both caches belong to the calling thread;
|
|
37
|
+
* strings are built with `PyUnicode_New` plus a copy. Non-ASCII UTF-8,
|
|
38
|
+
which simdjson has already validated, is transcoded without
|
|
39
|
+
revalidation: scalar code for short strings, simdutf for long ones;
|
|
40
|
+
* on builds that use the GIL, the cyclic GC is paused while objects are built.
|
|
41
|
+
3. Integers that do not fit in 64 bits stay on the tape as digit strings and
|
|
42
|
+
become exact Python ints (orjson turns them into floats).
|
|
43
|
+
|
|
44
|
+
`NaN`, `Infinity`, and `-Infinity` parse as floats, matching `json.loads`.
|
|
45
|
+
simdjson is built with `SIMDJSON_ENABLE_NAN_INF`, so any capitalization of
|
|
46
|
+
`nan`, `inf`, and `infinity` is also accepted (Python's parser only allows
|
|
47
|
+
the three spellings above). A number that overflows a double becomes
|
|
48
|
+
`inf`, via `json.loads`, because simdjson still rejects it. An unpaired
|
|
49
|
+
surrogate escape (`"\ud800"`) is also rejected by simdjson; `json.loads`
|
|
50
|
+
accepts it, so `loads` returns that string. A leading UTF-8 BOM is accepted.
|
|
51
|
+
|
|
52
|
+
## Build and test
|
|
53
|
+
|
|
54
|
+
Python 3.10 or newer, and a C++17 compiler (clang or GCC). The simdjson 5.0.1
|
|
55
|
+
and simdutf 9.2.1 amalgamations are already in `vendor/`.
|
|
56
|
+
|
|
57
|
+
pip:
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
python3 -m venv .venv
|
|
61
|
+
. .venv/bin/activate
|
|
62
|
+
python -m pip install -U pip setuptools
|
|
63
|
+
python -m pip install -e ".[test]"
|
|
64
|
+
pytest tests
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
uv:
|
|
68
|
+
|
|
69
|
+
```sh
|
|
70
|
+
uv venv
|
|
71
|
+
. .venv/bin/activate
|
|
72
|
+
uv pip install -e ".[test]"
|
|
73
|
+
pytest tests
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Either one builds the extension and makes `import fastsimdjson` work in the
|
|
77
|
+
virtualenv. uv uses the `setuptools` build requirement from `pyproject.toml`,
|
|
78
|
+
so it does not need a separate setuptools install for this path.
|
|
79
|
+
|
|
80
|
+
To compile the extension in the tree instead, install setuptools and pytest
|
|
81
|
+
into the same virtualenv, then:
|
|
82
|
+
|
|
83
|
+
```sh
|
|
84
|
+
python -m pip install setuptools pytest
|
|
85
|
+
python setup.py build_ext --inplace
|
|
86
|
+
PYTHONPATH=src python -m pytest tests
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
uv pip install setuptools pytest
|
|
91
|
+
python setup.py build_ext --inplace
|
|
92
|
+
PYTHONPATH=src python -m pytest tests
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Recent setuptools copies the `.so` next to `src/fastsimdjson.cpp`, which is
|
|
96
|
+
why `PYTHONPATH=src` is required for the in-place build.
|
|
97
|
+
|
|
98
|
+
`tests/test_loads.py` compares `loads` with `json.loads` on types and key
|
|
99
|
+
order. It covers scalars, integers past 64 bits, UTF-8 strings at every
|
|
100
|
+
length from 0 to 199, the key cache, random documents, rejected input, deep
|
|
101
|
+
nesting, padding at a page boundary, a saturated array count, reference
|
|
102
|
+
counts, and release of a parser that has grown past 64 MB. The corpus test
|
|
103
|
+
is skipped until simdjson-data is checked out beside the project:
|
|
104
|
+
|
|
105
|
+
```sh
|
|
106
|
+
git clone --depth 1 https://github.com/simdjson/simdjson-data.git
|
|
107
|
+
pytest tests
|
|
108
|
+
# or: JSONDIR=/path/to/jsonexamples pytest tests
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The suite builds an ~80 MB document and a list of 16,777,221 integers, so
|
|
112
|
+
give it some RAM.
|
|
113
|
+
|
|
114
|
+
To time `loads` against `json.loads` and orjson on those files:
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
python -m pip install -e ".[bench]"
|
|
118
|
+
python bench.py
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
```sh
|
|
122
|
+
uv pip install -e ".[bench]"
|
|
123
|
+
python bench.py
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## Benchmarks
|
|
127
|
+
|
|
128
|
+
Full materialization of `simdjson-data` on an Intel Xeon Gold 6548N, pinned
|
|
129
|
+
to one core. Python 3.12.13, GCC, `json.loads`, orjson 3.12.0, yyjson 4.0.6,
|
|
130
|
+
cysimdjson 26.27 (simdjson 3.8.0, AVX-512), pysimdjson 7.0.2 (icelake,
|
|
131
|
+
AVX-512). Times are microseconds, the best of several runs. Speedup is
|
|
132
|
+
orjson time / fastsimdjson time.
|
|
133
|
+
|
|
134
|
+
cysimdjson is `JSONParser.parse(data).export()` and pysimdjson is
|
|
135
|
+
`Parser.parse(data, recursive=True)`. `yyjson.loads` is `Document.as_obj`.
|
|
136
|
+
fastsimdjson, `json.loads`, cysimdjson, and pysimdjson matched orjson on
|
|
137
|
+
every file, including types and key order.
|
|
138
|
+
|
|
139
|
+
Geomean speedup of fastsimdjson: 3.38× over `json.loads`, 1.27× over orjson,
|
|
140
|
+
1.39× over yyjson, 1.83× over cysimdjson, and 1.79× over pysimdjson. It was
|
|
141
|
+
the fastest on 21 files. `numbers.json`, a flat array of floats, is the loss
|
|
142
|
+
against orjson (0.95×): the time is simdjson's float parser plus allocating
|
|
143
|
+
Python floats. `json.loads` is slowest on every file; the gap is largest on
|
|
144
|
+
`canada.json` (about 5.8×).
|
|
145
|
+
|
|
146
|
+
yyjson 4.0.6 returns non-ASCII strings as the raw UTF-8 bytes stored in a
|
|
147
|
+
Latin-1 `str`. Twelve files therefore do not match orjson (the twitter
|
|
148
|
+
files, `citm_catalog`, `gsoc-2018`, `github_events`, `random`, `repeat`,
|
|
149
|
+
`semanticscholar-corpus`, and `update-center`). Those yyjson times skip a
|
|
150
|
+
real UTF-8 decode. fastsimdjson is still ahead on the ten files where the
|
|
151
|
+
values match.
|
|
152
|
+
|
|
153
|
+
| file | KB | json | orjson | fast | yyjson | cys | psy | speedup |
|
|
154
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
155
|
+
| apache_builds.json | 124 | 383.2 | 180.7 | 153.9 | 205.9 | 263.1 | 264.8 | 1.17 |
|
|
156
|
+
| canada.json | 2198 | 30404.4 | 6518.2 | 5248.1 | 5552.2 | 7894.4 | 7348.4 | 1.24 |
|
|
157
|
+
| citm_catalog.json | 1687 | 7295.1 | 2934.6 | 1775.7 | 2554.0 | 4321.0 | 4206.3 | 1.65 |
|
|
158
|
+
| github_events.json | 64 | 216.1 | 68.5 | 55.6 | 82.8 | 98.5 | 97.7 | 1.23 |
|
|
159
|
+
| google_maps_api_compact_response.json | 12 | 80.3 | 39.7 | 33.6 | 47.3 | 55.6 | 56.5 | 1.18 |
|
|
160
|
+
| google_maps_api_response.json | 25 | 93.6 | 43.4 | 34.8 | 52.1 | 56.9 | 57.7 | 1.25 |
|
|
161
|
+
| gsoc-2018.json | 3250 | 7039.9 | 3889.8 | 2280.5 | 2968.0 | 3569.8 | 3611.0 | 1.71 |
|
|
162
|
+
| instruments.json | 215 | 848.4 | 319.3 | 239.1 | 405.1 | 478.6 | 500.7 | 1.34 |
|
|
163
|
+
| marine_ik.json | 2914 | 23662.6 | 9301.2 | 7272.7 | 8548.5 | 12315.3 | 11046.9 | 1.28 |
|
|
164
|
+
| mesh.json | 707 | 5093.0 | 1712.2 | 1493.1 | 1685.6 | 2269.3 | 2074.4 | 1.15 |
|
|
165
|
+
| mesh.pretty.json | 1540 | 8382.9 | 2435.5 | 1674.3 | 2185.0 | 2548.5 | 2309.9 | 1.45 |
|
|
166
|
+
| numbers.json | 147 | 914.3 | 221.4 | 232.3 | 260.4 | 299.2 | 276.2 | 0.95 |
|
|
167
|
+
| random.json | 499 | 3107.1 | 1522.5 | 1292.4 | 1632.5 | 2571.5 | 2390.8 | 1.18 |
|
|
168
|
+
| repeat.json | 11 | 43.9 | 15.3 | 13.0 | 16.7 | 30.5 | 24.1 | 1.18 |
|
|
169
|
+
| semanticscholar-corpus.json | 8392 | 45587.1 | 20634.5 | 14113.1 | 18832.2 | 28079.0 | 26477.3 | 1.46 |
|
|
170
|
+
| tree-pretty.json | 34 | 117.5 | 43.5 | 36.3 | 62.5 | 63.5 | 65.4 | 1.20 |
|
|
171
|
+
| twitter.json | 617 | 2798.4 | 1017.4 | 715.4 | 1168.0 | 1771.2 | 1825.1 | 1.42 |
|
|
172
|
+
| twitter_api_compact_response.json | 10 | 45.0 | 14.8 | 12.1 | 19.3 | 22.6 | 22.7 | 1.22 |
|
|
173
|
+
| twitter_api_response.json | 15 | 60.3 | 17.4 | 13.9 | 23.4 | 26.2 | 26.4 | 1.25 |
|
|
174
|
+
| twitter_timeline.json | 41 | 176.7 | 61.2 | 52.6 | 82.8 | 110.9 | 114.6 | 1.16 |
|
|
175
|
+
| twitterescaped.json | 549 | 2308.5 | 1021.7 | 801.4 | 1184.0 | 1961.0 | 2030.3 | 1.27 |
|
|
176
|
+
| update-center.json | 521 | 2675.8 | 1363.6 | 1119.1 | 1509.2 | 1982.1 | 2057.0 | 1.22 |
|
|
177
|
+
|
|
178
|
+
## Limitations
|
|
179
|
+
|
|
180
|
+
* Only `loads` is implemented; there is no `dumps`.
|
|
181
|
+
* The simdjson parser and the key and string caches are thread-local.
|
|
182
|
+
`release()` frees the parser and the cached strings retained by the calling
|
|
183
|
+
thread. A parser that grows past 64 MB is freed on its own at the end of
|
|
184
|
+
that call; its caches stay. A thread that exits without `release()` leaves
|
|
185
|
+
its cached strings behind. The module is marked free-threading compatible
|
|
186
|
+
(`Py_MOD_GIL_NOT_USED` on Python 3.13 and newer), so importing it on a
|
|
187
|
+
free-threaded build does not re-enable the GIL. Subinterpreters are not
|
|
188
|
+
supported. On a free-threaded build, `bytearray` and `memoryview` inputs
|
|
189
|
+
are copied before parsing.
|
|
190
|
+
* A document simdjson rejects is reparsed with `json.loads`. `loads` returns
|
|
191
|
+
that value when `json.loads` accepts it, which is how overflow to infinity
|
|
192
|
+
is handled. An exception is raised only when `json.loads` also fails.
|
|
193
|
+
`JSONDecodeError` is re-raised as `fastsimdjson.JSONDecodeError` with the
|
|
194
|
+
same message, document, and position. Any other exception propagates.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "fastsimdjson"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Fast JSON parsing for Python, built on simdjson"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{name = "Daniel Lemire", email = "daniel@lemire.me"}
|
|
12
|
+
]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
license = "Apache-2.0"
|
|
15
|
+
keywords = ["json", "simdjson", "parser", "simd", "performance"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Programming Language :: C++",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Programming Language :: Python :: 3.14",
|
|
26
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
27
|
+
"Topic :: Software Development :: Libraries",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/simdjson/fastpysimdjson"
|
|
32
|
+
Repository = "https://github.com/simdjson/fastpysimdjson"
|
|
33
|
+
Issues = "https://github.com/simdjson/fastpysimdjson/issues"
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
test = ["pytest"]
|
|
37
|
+
bench = ["orjson"]
|
|
38
|
+
|
|
39
|
+
[tool.cibuildwheel]
|
|
40
|
+
skip = ["pp*"]
|
|
41
|
+
test-requires = ["pytest"]
|
|
42
|
+
test-command = "pytest {project}/tests -q"
|
|
43
|
+
|
|
44
|
+
[tool.cibuildwheel.macos]
|
|
45
|
+
# Build for Intel, Apple Silicon, and Universal2 binaries
|
|
46
|
+
archs = ["x86_64", "universal2", "arm64"]
|
|
47
|
+
|
|
48
|
+
[tool.cibuildwheel.linux]
|
|
49
|
+
# Build for the runner's native architecture (x86_64 and aarch64 runners
|
|
50
|
+
# are provided separately in the CI matrix, so each builds natively).
|
|
51
|
+
archs = ["native"]
|
|
52
|
+
|
|
53
|
+
[tool.cibuildwheel.windows]
|
|
54
|
+
# Build for 64-bit Windows (32-bit is rarely needed nowadays)
|
|
55
|
+
archs = ["AMD64"]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from setuptools import setup, Extension
|
|
2
|
+
from setuptools.command.build_ext import build_ext
|
|
3
|
+
|
|
4
|
+
# SIMDJSON_ENABLE_NAN_INF: accept NaN/Infinity (any case of nan, inf,
|
|
5
|
+
# and infinity). Off in the amalgamation unless the build defines it.
|
|
6
|
+
DEFINES = [("NDEBUG", None), ("SIMDJSON_ENABLE_NAN_INF", "1")]
|
|
7
|
+
|
|
8
|
+
ext = Extension(
|
|
9
|
+
"fastsimdjson",
|
|
10
|
+
sources=["src/fastsimdjson.cpp", "vendor/simdjson.cpp", "vendor/simdutf.cpp"],
|
|
11
|
+
include_dirs=["vendor"],
|
|
12
|
+
language="c++",
|
|
13
|
+
define_macros=DEFINES,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BuildExt(build_ext):
|
|
18
|
+
def build_extensions(self):
|
|
19
|
+
if self.compiler.compiler_type == "msvc":
|
|
20
|
+
args = ["/std:c++17", "/O2", "/EHsc", "/utf-8"]
|
|
21
|
+
else:
|
|
22
|
+
args = ["-std=c++17", "-O3"]
|
|
23
|
+
for e in self.extensions:
|
|
24
|
+
e.extra_compile_args = args
|
|
25
|
+
super().build_extensions()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
setup(ext_modules=[ext], cmdclass={"build_ext": BuildExt})
|