cxjson 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cxjson-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 James Francis
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
cxjson-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,267 @@
1
+ Metadata-Version: 2.4
2
+ Name: cxjson
3
+ Version: 0.1.0
4
+ Summary: Ultra-fast pure-C JSON library for Python with SIMD, CPython split-table dicts, and adaptive parallelism
5
+ Author: James Francis
6
+ License: MIT License
7
+
8
+ Copyright (c) 2026 James Francis
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
27
+
28
+ Project-URL: Homepage, https://github.com/jamesfrancis2004/cxjson
29
+ Project-URL: Repository, https://github.com/jamesfrancis2004/cxjson
30
+ Project-URL: Issues, https://github.com/jamesfrancis2004/cxjson/issues
31
+ Keywords: json,fast,simd,c-extension,parser,serializer,orjson
32
+ Classifier: Development Status :: 4 - Beta
33
+ Classifier: Intended Audience :: Developers
34
+ Classifier: Programming Language :: Python :: 3
35
+ Classifier: Programming Language :: Python :: 3.12
36
+ Classifier: Programming Language :: Python :: 3.13
37
+ Classifier: Programming Language :: Python :: 3.14
38
+ Classifier: Programming Language :: C
39
+ Classifier: Operating System :: OS Independent
40
+ Classifier: Typing :: Typed
41
+ Requires-Python: >=3.9
42
+ Description-Content-Type: text/markdown
43
+ License-File: LICENSE
44
+ Dynamic: license-file
45
+
46
+ # cxjson
47
+
48
+ > **I tasked Gemini with building a JSON library in C to beat `orjson` — the fastest JSON library in Python. It didn't just beat `orjson`; it swept 56/56 benchmarks across 28 real-world datasets and used 4.7×–10.6× less RAM.**
49
+
50
+ [![CI](https://github.com/jamesfrancis2004/cxjson/actions/workflows/ci.yml/badge.svg)](https://github.com/jamesfrancis2004/cxjson/actions/workflows/ci.yml)
51
+ [![Python 3.12+](https://img.shields.io/badge/python-3.12%20|%203.13%20|%203.14-blue.svg)](#)
52
+ [![Pure C](https://img.shields.io/badge/implementation-pure%20C%20%2B%20SIMD-orange.svg)](#)
53
+ [![RFC 8259](https://img.shields.io/badge/RFC%208259-strict%20conformance-brightgreen.svg)](#)
54
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
55
+
56
+ ---
57
+
58
+ ## The Challenge
59
+
60
+ For years, [`orjson`](https://github.com/ijl/orjson) (written in Rust) has been the undisputed speed king of Python JSON libraries. Every time someone asks *"how do I make JSON faster in Python?"*, the answer is always *"just use `orjson`."*
61
+
62
+ I wanted to see what would happen if I tasked Gemini with building a zero-dependency pure-C Python extension from scratch—with one simple goal: **beat `orjson` across real-world JSON workloads, without cheating on RFC 8259 correctness, memory safety, or API compatibility.**
63
+
64
+ The result is **`cxjson`**.
65
+
66
+ ### Headline Results (Apple M2, Python 3.12.3)
67
+
68
+ - **56 / 56 wins (100%) vs `orjson`** across **28 real-world JSON datasets** (`twitter`, `citm_catalog`, `canada`, `gsoc-2018`, `usgs_earthquakes`, `pypi_numpy`, `npm_typescript`, `semanticscholar-corpus`, and 20 more)
69
+ - **1.79× faster `dumps`** and **1.46× faster `loads`** (geo-mean vs `orjson`)
70
+ - **4.7×–10.6× lower peak memory** during `loads` than `orjson`
71
+ - **4.0+ GB/s** adaptive `dumps` (**2.8 GB/s** single-threaded) and **~1.0 GB/s** `loads` — beating `orjson`, `msgspec`, `jiter`, `ujson`, `python-rapidjson`, and Python's stdlib `json`
72
+
73
+ ---
74
+
75
+ ## Python JSON Library Benchmarks (Apple M2 / ARM64)
76
+
77
+ All charts and tables below were measured locally on an **Apple M2 (ARM64)** with **Python 3.12.3** across 8 representative real-world JSON datasets (`twitter.json`, `citm_catalog.json`, `canada.json`, `gsoc-2018.json`, `usgs_earthquakes_week.json`, `pypi_numpy.json`, `semanticscholar-corpus.json`, `npm_typescript.json`).
78
+
79
+ > **Note on x86_64:** `cxjson` includes runtime-dispatched SSE2, AVX2, and AVX-512BW kernels for x86_64, but all benchmarks below were measured on Apple Silicon (ARM64). We haven't thoroughly benchmarked or tuned bare-metal x86_64 performance yet, so results on x86_64 will vary and there are likely still x86-specific optimizations on the table (contributions and x86 benchmarks are very welcome!).
80
+
81
+ ![Python JSON Libraries Overview](docs/images/python_libraries_overview.png)
82
+
83
+ ### Per-Dataset Breakdown
84
+
85
+ ![Per-Dataset Throughput Comparison](docs/images/python_per_dataset.png)
86
+
87
+ #### Serialization (`dumps` / `encode`) — MB/s (higher is better)
88
+
89
+ | Dataset | Size | **`cxjson`** | **`cxjson (1T)`** | `orjson` | `msgspec` | `ujson` | `rapidjson` | `json` (stdlib) |
90
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|
91
+ | `twitter.json` | 0.60 MB | 3,514 | **3,516** | 2,738 | 1,705 | 524 | 316 | 321 |
92
+ | `citm_catalog.json` | 1.65 MB | 1,872 | **2,161** | 1,382 | 1,208 | 263 | 242 | 178 |
93
+ | `canada.json` | 2.15 MB | **2,573** | 1,611 | 1,208 | 723 | 190 | 53 | 56 |
94
+ | `gsoc-2018.json` | 3.17 MB | **6,268** | 4,917 | 4,603 | 2,427 | 978 | 392 | 451 |
95
+ | `usgs_earthquakes_week.json` | 1.28 MB | **6,463** | 2,179 | 1,341 | 996 | 309 | 211 | 180 |
96
+ | `pypi_numpy.json` | 3.61 MB | **4,251** | 3,367 | 2,810 | 1,566 | 588 | 349 | 334 |
97
+ | `semanticscholar-corpus.json` | 8.20 MB | **5,558** | 3,428 | 1,766 | 1,447 | 489 | 332 | 306 |
98
+ | `npm_typescript.json` | 15.22 MB | **4,596** | 2,474 | 1,362 | 1,293 | 389 | 296 | 278 |
99
+ | **Geo-Mean (MB/s)** | — | **4,064** | **2,797** | **1,928** | **1,342** | **415** | **241** | **228** |
100
+
101
+ #### Deserialization (`loads` / `decode`) — MB/s (higher is better)
102
+
103
+ | Dataset | Size | **`cxjson`** | `orjson` | `msgspec` | `jiter` | `ujson` | `json` (stdlib) | `rapidjson` |
104
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|
105
+ | `twitter.json` | 0.60 MB | **1,087** | 753 | 599 | 492 | 441 | 348 | 214 |
106
+ | `citm_catalog.json` | 1.65 MB | **1,343** | 717 | 614 | 590 | 534 | 401 | 300 |
107
+ | `canada.json` | 2.15 MB | **494** | 358 | 297 | 314 | 207 | 99 | 94 |
108
+ | `gsoc-2018.json` | 3.17 MB | **1,776** | 1,181 | 1,330 | 1,443 | 782 | 714 | 389 |
109
+ | `usgs_earthquakes_week.json` | 1.28 MB | **962** | 417 | 373 | 305 | 243 | 171 | 154 |
110
+ | `pypi_numpy.json` | 3.61 MB | **1,641** | 665 | 663 | 598 | 410 | 339 | 240 |
111
+ | `semanticscholar-corpus.json` | 8.20 MB | **926** | 448 | 502 | 402 | 330 | 319 | 178 |
112
+ | `npm_typescript.json` | 15.22 MB | **708** | 383 | 377 | 405 | 253 | 263 | 193 |
113
+ | **Geo-Mean (MB/s)** | — | **1,036** | **568** | **536** | **500** | **365** | **288** | **204** |
114
+
115
+ ---
116
+
117
+ ## Did We Overfit? Testing 28 Real-World Datasets
118
+
119
+ To make sure we hadn't accidentally hyper-optimized for a handful of classic benchmark files, we expanded the benchmark suite to **28 diverse datasets**—including live production JSON payloads pulled directly from the PyPI JSON API (`requests`, `pandas`, `numpy`), npm registry (`react`, `typescript`, `lodash`), USGS Earthquake GeoJSON feeds, Nobel Prize API, Semantic Scholar corpus, GitHub events, and deeply nested pretty-printed trees.
120
+
121
+ `cxjson` swept **56 / 56 (100%)** against `orjson` on Apple M2:
122
+
123
+ ![28 Real-World Datasets vs orjson](docs/images/benchmark_28_datasets.png)
124
+
125
+ ---
126
+
127
+ ## Peak Memory Footprint
128
+
129
+ Speed is only half the story—when parsing large API payloads in production, memory spikes are often what OOM-kill Python workers. Because `cxjson` builds CPython split-table dicts sharing a single `PyDictKeysObject` across repeated object schemas, peak allocation during `loads()` drops by **4.7×–10.6×** compared to `orjson`:
130
+
131
+ ![Peak Memory Comparison](docs/images/memory_comparison.png)
132
+
133
+ | Dataset | `cxjson` Peak `loads` | `orjson` Peak `loads` | Reduction |
134
+ |---|---:|---:|---:|
135
+ | `twitter.json` | **0.85 MB** | 8.24 MB | **9.7× less** |
136
+ | `citm_catalog.json` | **2.16 MB** | 22.85 MB | **10.6× less** |
137
+ | `gsoc-2018.json` | **4.43 MB** | 42.96 MB | **9.7× less** |
138
+ | `canada.json` | **6.94 MB** | 32.57 MB | **4.7× less** |
139
+ | `semanticscholar-corpus.json` | **17.41 MB** | 121.45 MB | **7.0× less** |
140
+
141
+ ---
142
+
143
+ ## NumPy Serialization (`OPT_SERIALIZE_NUMPY`)
144
+
145
+ `cxjson` serializes NumPy arrays directly from their raw C memory via Python's PEP 3118 buffer protocol—**zero NumPy build dependency** (works across NumPy 1.x and 2.x) and supports `int16`/`uint16` plus non-contiguous strided/transposed views.
146
+
147
+ | NumPy Array (Apple M2, NumPy 2.1.2) | **`cxjson`** | `orjson` | Speedup |
148
+ |---|---:|---:|---:|
149
+ | `float64` (100K 1D) | **2,061 MB/s** | 822 MB/s | **2.51×** |
150
+ | `float64` (500×200 2D) | **2,060 MB/s** | 804 MB/s | **2.56×** |
151
+ | `float32` (100K 1D) | **1,407 MB/s** | 446 MB/s | **3.16×** |
152
+ | `int64` (100K 1D) | **1,747 MB/s** | 1,476 MB/s | **1.18×** |
153
+ | `int32` (100K 1D) | **1,410 MB/s** | 1,262 MB/s | **1.12×** |
154
+ | `uint8` (200K 1D) | **7,551 MB/s** | 968 MB/s | **7.80×** |
155
+ | `bool` (200K 1D) | **8,631 MB/s** | 4,130 MB/s | **2.09×** |
156
+ | **Geo-Mean** | — | — | **2.38×** |
157
+
158
+ ---
159
+
160
+ ## How Is `cxjson` This Fast?
161
+
162
+ There is no magic—just relentless elimination of per-character branches, redundant copies, and CPython object allocations:
163
+
164
+ 1. **Runtime-Dispatched SIMD (ARM64 NEON, x86_64 AVX2, SSE2):** String scanning, quote/escape detection, UTF-8 ASCII runs, and Latin-1 narrowing process 16–32 bytes per cycle. On x86_64, `cxjson` detects AVX2 vs SSE2 at import time via CPUID so a single wheel runs optimally on any CPU.
165
+ 2. **CPython Split-Table Dicts (Python 3.12, 3.13, & 3.14):** When parsing arrays of objects with repeated schemas (like REST API responses or GeoJSON features), `cxjson` learns the key schema and builds CPython split-table dicts sharing a single `PyDictKeysObject`. This eliminates per-dict hash table allocations and is why `loads` uses **7×–10.5× less peak memory** than `orjson` while running **1.5×–2.5× faster**.
166
+ 3. **Key & Value-String Caching:** L1-resident direct-mapped/2-way caches for ASCII dict keys (`1..64B`), UTF-8 keys, and repeated ASCII string values (`2..48B`), plus singletons for `""`, 1-char ASCII strings, and small integers.
167
+ 4. **State-of-the-Art Number Formatting & Parsing:** Vendored [`zmij`](src/c/zmij) (float → string) and [`jeaiii`](src/c/jeaiii) (int → string) for serialization, paired with 8-digit SWAR mantissa parsing + Clinger's fast path + Eisel-Lemire for bit-exact IEEE-754 float parsing.
168
+ 5. **Adaptive Multi-Threaded `dumps` (with `pthread_atfork()` Safety):** For payloads ≥ 512 KB of pure built-in types, `cxjson` scales worker threads by payload size while holding the GIL (workers read immutable Python memory without touching refcounts or C-API). It is safe across `os.fork()`, automatically disabled on free-threaded Python (`Py_GIL_DISABLED`), and configurable via `CXJSON_MAX_THREADS=1`.
169
+
170
+ ---
171
+
172
+ ## Safety & Correctness
173
+
174
+ Speed means nothing if a parser crashes on untrusted input or silently accepts invalid JSON:
175
+
176
+ - **Strict RFC 8259 Conformance:** Rejects trailing commas, leading zeros, invalid UTF-8, lone surrogates, unescaped control characters, `NaN`/`Infinity` (by default), and trailing garbage—verified against 44 valid + 67 invalid RFC 8259 test cases matching `orjson`.
177
+ - **Fuzz Tested:** Verified against **10,000 mutated-byte fuzz inputs** and **3,000 randomized structured round-trips** matching `orjson` 100%.
178
+ - **Version-Guarded CPython Internals:** Split-table dicts and direct `PyLongObject` digit reads are strictly `#if`-guarded by `PY_VERSION_HEX` and `!Py_GIL_DISABLED`, falling back cleanly to the public CPython C-API on unsupported versions or free-threaded builds.
179
+ - **Zero External Dependencies:** 100% pure C, zero C++ runtime linkage, zero warnings under `-Wall -Wextra -Werror`.
180
+
181
+ ---
182
+
183
+ ## Quickstart & Usage
184
+
185
+ ### Build & Install Locally
186
+
187
+ ```bash
188
+ git clone https://github.com/jamesfrancis2004/cxjson.git
189
+ cd cxjson
190
+ pip install -e .
191
+ ```
192
+
193
+ ### Basic API
194
+
195
+ ```python
196
+ import cxjson
197
+
198
+ # Serialize to UTF-8 bytes (fastest, drop-in compatible with orjson.dumps)
199
+ payload: bytes = cxjson.dumps({"hello": "world", "nums": [1, 2, 3.14]})
200
+ # b'{"hello":"world","nums":[1,2,3.14]}'
201
+
202
+ # Deserialize from bytes, bytearray, memoryview, or str
203
+ obj = cxjson.loads(payload)
204
+
205
+ # Serialize to str (like stdlib json.dumps)
206
+ text: str = cxjson.dumps_str(obj)
207
+ ```
208
+
209
+ ### Drop-In FastAPI / Starlette Response
210
+
211
+ ```python
212
+ from starlette.responses import Response
213
+ import cxjson
214
+
215
+ class CXJSONResponse(Response):
216
+ media_type = "application/json"
217
+
218
+ def render(self, content) -> bytes:
219
+ return cxjson.dumps(content)
220
+ ```
221
+
222
+ > **Tip for multi-worker servers (Uvicorn / Gunicorn):** Set `CXJSON_MAX_THREADS=1` so worker processes don't compete for cores with internal serialization threads—even at 1 thread, `cxjson` serializes at **~2.8 GB/s**.
223
+
224
+ ### Running Tests & Benchmarks
225
+
226
+ ```bash
227
+ # Build C extension in-place
228
+ make build
229
+
230
+ # Run serializer, orjson-compatibility, RFC 8259 conformance, fork, and fuzz suites
231
+ make test
232
+
233
+ # Run the 28-dataset benchmark against orjson
234
+ PYTHONPATH=src python3 tests/bench_comprehensive.py
235
+ ```
236
+
237
+ ---
238
+
239
+ ## Project Layout
240
+
241
+ ```text
242
+ cxjson/
243
+ ├── docs/images/ # Benchmark bar charts
244
+ ├── src/
245
+ │ ├── c/
246
+ │ │ ├── _cxjson_module.c # CPython extension entrypoint
247
+ │ │ ├── serializer.c # SIMD + zmij/jeaiii serializer
248
+ │ │ ├── deserializer.c # SIMD + split-table dict deserializer
249
+ │ │ ├── cjson_simd.c/h # NEON / AVX2 / SSE2 kernels + runtime dispatch
250
+ │ │ ├── key_cache.h # Key/value caches & split-table dict builders
251
+ │ │ ├── numparse.h # SWAR + Clinger + Eisel-Lemire number parser
252
+ │ │ ├── parallel_ser.h # Adaptive cross-platform thread pool + atfork
253
+ │ │ ├── utf8.h # SIMD UTF-8 validation & transcoding
254
+ │ │ ├── jeaiii/ # Fast integer-to-ASCII
255
+ │ │ └── zmij/ # Fast double-to-ASCII
256
+ │ └── cxjson/ # Python package, PEP 561 stubs (__init__.pyi, py.typed)
257
+ └── tests/ # Unit, compatibility, RFC 8259 conformance/fuzz, & benchmarks
258
+ ```
259
+
260
+ ---
261
+
262
+ ## Contributing & License
263
+
264
+ Contributions are welcome—see [CONTRIBUTING.md](CONTRIBUTING.md) for build instructions, C guidelines, and how to run the fuzz/benchmark suites.
265
+
266
+ Licensed under the [MIT License](LICENSE).
267
+
cxjson-0.1.0/README.md ADDED
@@ -0,0 +1,222 @@
1
+ # cxjson
2
+
3
+ > **I tasked Gemini with building a JSON library in C to beat `orjson` — the fastest JSON library in Python. It didn't just beat `orjson`; it swept 56/56 benchmarks across 28 real-world datasets and used 4.7×–10.6× less RAM.**
4
+
5
+ [![CI](https://github.com/jamesfrancis2004/cxjson/actions/workflows/ci.yml/badge.svg)](https://github.com/jamesfrancis2004/cxjson/actions/workflows/ci.yml)
6
+ [![Python 3.12+](https://img.shields.io/badge/python-3.12%20|%203.13%20|%203.14-blue.svg)](#)
7
+ [![Pure C](https://img.shields.io/badge/implementation-pure%20C%20%2B%20SIMD-orange.svg)](#)
8
+ [![RFC 8259](https://img.shields.io/badge/RFC%208259-strict%20conformance-brightgreen.svg)](#)
9
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
10
+
11
+ ---
12
+
13
+ ## The Challenge
14
+
15
+ For years, [`orjson`](https://github.com/ijl/orjson) (written in Rust) has been the undisputed speed king of Python JSON libraries. Every time someone asks *"how do I make JSON faster in Python?"*, the answer is always *"just use `orjson`."*
16
+
17
+ I wanted to see what would happen if I tasked Gemini with building a zero-dependency pure-C Python extension from scratch—with one simple goal: **beat `orjson` across real-world JSON workloads, without cheating on RFC 8259 correctness, memory safety, or API compatibility.**
18
+
19
+ The result is **`cxjson`**.
20
+
21
+ ### Headline Results (Apple M2, Python 3.12.3)
22
+
23
+ - **56 / 56 wins (100%) vs `orjson`** across **28 real-world JSON datasets** (`twitter`, `citm_catalog`, `canada`, `gsoc-2018`, `usgs_earthquakes`, `pypi_numpy`, `npm_typescript`, `semanticscholar-corpus`, and 20 more)
24
+ - **1.79× faster `dumps`** and **1.46× faster `loads`** (geo-mean vs `orjson`)
25
+ - **4.7×–10.6× lower peak memory** during `loads` than `orjson`
26
+ - **4.0+ GB/s** adaptive `dumps` (**2.8 GB/s** single-threaded) and **~1.0 GB/s** `loads` — beating `orjson`, `msgspec`, `jiter`, `ujson`, `python-rapidjson`, and Python's stdlib `json`
27
+
28
+ ---
29
+
30
+ ## Python JSON Library Benchmarks (Apple M2 / ARM64)
31
+
32
+ All charts and tables below were measured locally on an **Apple M2 (ARM64)** with **Python 3.12.3** across 8 representative real-world JSON datasets (`twitter.json`, `citm_catalog.json`, `canada.json`, `gsoc-2018.json`, `usgs_earthquakes_week.json`, `pypi_numpy.json`, `semanticscholar-corpus.json`, `npm_typescript.json`).
33
+
34
+ > **Note on x86_64:** `cxjson` includes runtime-dispatched SSE2, AVX2, and AVX-512BW kernels for x86_64, but all benchmarks below were measured on Apple Silicon (ARM64). We haven't thoroughly benchmarked or tuned bare-metal x86_64 performance yet, so results on x86_64 will vary and there are likely still x86-specific optimizations on the table (contributions and x86 benchmarks are very welcome!).
35
+
36
+ ![Python JSON Libraries Overview](docs/images/python_libraries_overview.png)
37
+
38
+ ### Per-Dataset Breakdown
39
+
40
+ ![Per-Dataset Throughput Comparison](docs/images/python_per_dataset.png)
41
+
42
+ #### Serialization (`dumps` / `encode`) — MB/s (higher is better)
43
+
44
+ | Dataset | Size | **`cxjson`** | **`cxjson (1T)`** | `orjson` | `msgspec` | `ujson` | `rapidjson` | `json` (stdlib) |
45
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|
46
+ | `twitter.json` | 0.60 MB | 3,514 | **3,516** | 2,738 | 1,705 | 524 | 316 | 321 |
47
+ | `citm_catalog.json` | 1.65 MB | 1,872 | **2,161** | 1,382 | 1,208 | 263 | 242 | 178 |
48
+ | `canada.json` | 2.15 MB | **2,573** | 1,611 | 1,208 | 723 | 190 | 53 | 56 |
49
+ | `gsoc-2018.json` | 3.17 MB | **6,268** | 4,917 | 4,603 | 2,427 | 978 | 392 | 451 |
50
+ | `usgs_earthquakes_week.json` | 1.28 MB | **6,463** | 2,179 | 1,341 | 996 | 309 | 211 | 180 |
51
+ | `pypi_numpy.json` | 3.61 MB | **4,251** | 3,367 | 2,810 | 1,566 | 588 | 349 | 334 |
52
+ | `semanticscholar-corpus.json` | 8.20 MB | **5,558** | 3,428 | 1,766 | 1,447 | 489 | 332 | 306 |
53
+ | `npm_typescript.json` | 15.22 MB | **4,596** | 2,474 | 1,362 | 1,293 | 389 | 296 | 278 |
54
+ | **Geo-Mean (MB/s)** | — | **4,064** | **2,797** | **1,928** | **1,342** | **415** | **241** | **228** |
55
+
56
+ #### Deserialization (`loads` / `decode`) — MB/s (higher is better)
57
+
58
+ | Dataset | Size | **`cxjson`** | `orjson` | `msgspec` | `jiter` | `ujson` | `json` (stdlib) | `rapidjson` |
59
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|
60
+ | `twitter.json` | 0.60 MB | **1,087** | 753 | 599 | 492 | 441 | 348 | 214 |
61
+ | `citm_catalog.json` | 1.65 MB | **1,343** | 717 | 614 | 590 | 534 | 401 | 300 |
62
+ | `canada.json` | 2.15 MB | **494** | 358 | 297 | 314 | 207 | 99 | 94 |
63
+ | `gsoc-2018.json` | 3.17 MB | **1,776** | 1,181 | 1,330 | 1,443 | 782 | 714 | 389 |
64
+ | `usgs_earthquakes_week.json` | 1.28 MB | **962** | 417 | 373 | 305 | 243 | 171 | 154 |
65
+ | `pypi_numpy.json` | 3.61 MB | **1,641** | 665 | 663 | 598 | 410 | 339 | 240 |
66
+ | `semanticscholar-corpus.json` | 8.20 MB | **926** | 448 | 502 | 402 | 330 | 319 | 178 |
67
+ | `npm_typescript.json` | 15.22 MB | **708** | 383 | 377 | 405 | 253 | 263 | 193 |
68
+ | **Geo-Mean (MB/s)** | — | **1,036** | **568** | **536** | **500** | **365** | **288** | **204** |
69
+
70
+ ---
71
+
72
+ ## Did We Overfit? Testing 28 Real-World Datasets
73
+
74
+ To make sure we hadn't accidentally hyper-optimized for a handful of classic benchmark files, we expanded the benchmark suite to **28 diverse datasets**—including live production JSON payloads pulled directly from the PyPI JSON API (`requests`, `pandas`, `numpy`), npm registry (`react`, `typescript`, `lodash`), USGS Earthquake GeoJSON feeds, Nobel Prize API, Semantic Scholar corpus, GitHub events, and deeply nested pretty-printed trees.
75
+
76
+ `cxjson` swept **56 / 56 (100%)** against `orjson` on Apple M2:
77
+
78
+ ![28 Real-World Datasets vs orjson](docs/images/benchmark_28_datasets.png)
79
+
80
+ ---
81
+
82
+ ## Peak Memory Footprint
83
+
84
+ Speed is only half the story—when parsing large API payloads in production, memory spikes are often what OOM-kill Python workers. Because `cxjson` builds CPython split-table dicts sharing a single `PyDictKeysObject` across repeated object schemas, peak allocation during `loads()` drops by **4.7×–10.6×** compared to `orjson`:
85
+
86
+ ![Peak Memory Comparison](docs/images/memory_comparison.png)
87
+
88
+ | Dataset | `cxjson` Peak `loads` | `orjson` Peak `loads` | Reduction |
89
+ |---|---:|---:|---:|
90
+ | `twitter.json` | **0.85 MB** | 8.24 MB | **9.7× less** |
91
+ | `citm_catalog.json` | **2.16 MB** | 22.85 MB | **10.6× less** |
92
+ | `gsoc-2018.json` | **4.43 MB** | 42.96 MB | **9.7× less** |
93
+ | `canada.json` | **6.94 MB** | 32.57 MB | **4.7× less** |
94
+ | `semanticscholar-corpus.json` | **17.41 MB** | 121.45 MB | **7.0× less** |
95
+
96
+ ---
97
+
98
+ ## NumPy Serialization (`OPT_SERIALIZE_NUMPY`)
99
+
100
+ `cxjson` serializes NumPy arrays directly from their raw C memory via Python's PEP 3118 buffer protocol—**zero NumPy build dependency** (works across NumPy 1.x and 2.x) and supports `int16`/`uint16` plus non-contiguous strided/transposed views.
101
+
102
+ | NumPy Array (Apple M2, NumPy 2.1.2) | **`cxjson`** | `orjson` | Speedup |
103
+ |---|---:|---:|---:|
104
+ | `float64` (100K 1D) | **2,061 MB/s** | 822 MB/s | **2.51×** |
105
+ | `float64` (500×200 2D) | **2,060 MB/s** | 804 MB/s | **2.56×** |
106
+ | `float32` (100K 1D) | **1,407 MB/s** | 446 MB/s | **3.16×** |
107
+ | `int64` (100K 1D) | **1,747 MB/s** | 1,476 MB/s | **1.18×** |
108
+ | `int32` (100K 1D) | **1,410 MB/s** | 1,262 MB/s | **1.12×** |
109
+ | `uint8` (200K 1D) | **7,551 MB/s** | 968 MB/s | **7.80×** |
110
+ | `bool` (200K 1D) | **8,631 MB/s** | 4,130 MB/s | **2.09×** |
111
+ | **Geo-Mean** | — | — | **2.38×** |
112
+
113
+ ---
114
+
115
+ ## How Is `cxjson` This Fast?
116
+
117
+ There is no magic—just relentless elimination of per-character branches, redundant copies, and CPython object allocations:
118
+
119
+ 1. **Runtime-Dispatched SIMD (ARM64 NEON, x86_64 AVX2, SSE2):** String scanning, quote/escape detection, UTF-8 ASCII runs, and Latin-1 narrowing process 16–32 bytes per cycle. On x86_64, `cxjson` detects AVX2 vs SSE2 at import time via CPUID so a single wheel runs optimally on any CPU.
120
+ 2. **CPython Split-Table Dicts (Python 3.12, 3.13, & 3.14):** When parsing arrays of objects with repeated schemas (like REST API responses or GeoJSON features), `cxjson` learns the key schema and builds CPython split-table dicts sharing a single `PyDictKeysObject`. This eliminates per-dict hash table allocations and is why `loads` uses **7×–10.5× less peak memory** than `orjson` while running **1.5×–2.5× faster**.
121
+ 3. **Key & Value-String Caching:** L1-resident direct-mapped/2-way caches for ASCII dict keys (`1..64B`), UTF-8 keys, and repeated ASCII string values (`2..48B`), plus singletons for `""`, 1-char ASCII strings, and small integers.
122
+ 4. **State-of-the-Art Number Formatting & Parsing:** Vendored [`zmij`](src/c/zmij) (float → string) and [`jeaiii`](src/c/jeaiii) (int → string) for serialization, paired with 8-digit SWAR mantissa parsing + Clinger's fast path + Eisel-Lemire for bit-exact IEEE-754 float parsing.
123
+ 5. **Adaptive Multi-Threaded `dumps` (with `pthread_atfork()` Safety):** For payloads ≥ 512 KB of pure built-in types, `cxjson` scales worker threads by payload size while holding the GIL (workers read immutable Python memory without touching refcounts or C-API). It is safe across `os.fork()`, automatically disabled on free-threaded Python (`Py_GIL_DISABLED`), and configurable via `CXJSON_MAX_THREADS=1`.
124
+
125
+ ---
126
+
127
+ ## Safety & Correctness
128
+
129
+ Speed means nothing if a parser crashes on untrusted input or silently accepts invalid JSON:
130
+
131
+ - **Strict RFC 8259 Conformance:** Rejects trailing commas, leading zeros, invalid UTF-8, lone surrogates, unescaped control characters, `NaN`/`Infinity` (by default), and trailing garbage—verified against 44 valid + 67 invalid RFC 8259 test cases matching `orjson`.
132
+ - **Fuzz Tested:** Verified against **10,000 mutated-byte fuzz inputs** and **3,000 randomized structured round-trips** matching `orjson` 100%.
133
+ - **Version-Guarded CPython Internals:** Split-table dicts and direct `PyLongObject` digit reads are strictly `#if`-guarded by `PY_VERSION_HEX` and `!Py_GIL_DISABLED`, falling back cleanly to the public CPython C-API on unsupported versions or free-threaded builds.
134
+ - **Zero External Dependencies:** 100% pure C, zero C++ runtime linkage, zero warnings under `-Wall -Wextra -Werror`.
135
+
136
+ ---
137
+
138
+ ## Quickstart & Usage
139
+
140
+ ### Build & Install Locally
141
+
142
+ ```bash
143
+ git clone https://github.com/jamesfrancis2004/cxjson.git
144
+ cd cxjson
145
+ pip install -e .
146
+ ```
147
+
148
+ ### Basic API
149
+
150
+ ```python
151
+ import cxjson
152
+
153
+ # Serialize to UTF-8 bytes (fastest, drop-in compatible with orjson.dumps)
154
+ payload: bytes = cxjson.dumps({"hello": "world", "nums": [1, 2, 3.14]})
155
+ # b'{"hello":"world","nums":[1,2,3.14]}'
156
+
157
+ # Deserialize from bytes, bytearray, memoryview, or str
158
+ obj = cxjson.loads(payload)
159
+
160
+ # Serialize to str (like stdlib json.dumps)
161
+ text: str = cxjson.dumps_str(obj)
162
+ ```
163
+
164
+ ### Drop-In FastAPI / Starlette Response
165
+
166
+ ```python
167
+ from starlette.responses import Response
168
+ import cxjson
169
+
170
+ class CXJSONResponse(Response):
171
+ media_type = "application/json"
172
+
173
+ def render(self, content) -> bytes:
174
+ return cxjson.dumps(content)
175
+ ```
176
+
177
+ > **Tip for multi-worker servers (Uvicorn / Gunicorn):** Set `CXJSON_MAX_THREADS=1` so worker processes don't compete for cores with internal serialization threads—even at 1 thread, `cxjson` serializes at **~2.8 GB/s**.
178
+
179
+ ### Running Tests & Benchmarks
180
+
181
+ ```bash
182
+ # Build C extension in-place
183
+ make build
184
+
185
+ # Run serializer, orjson-compatibility, RFC 8259 conformance, fork, and fuzz suites
186
+ make test
187
+
188
+ # Run the 28-dataset benchmark against orjson
189
+ PYTHONPATH=src python3 tests/bench_comprehensive.py
190
+ ```
191
+
192
+ ---
193
+
194
+ ## Project Layout
195
+
196
+ ```text
197
+ cxjson/
198
+ ├── docs/images/ # Benchmark bar charts
199
+ ├── src/
200
+ │ ├── c/
201
+ │ │ ├── _cxjson_module.c # CPython extension entrypoint
202
+ │ │ ├── serializer.c # SIMD + zmij/jeaiii serializer
203
+ │ │ ├── deserializer.c # SIMD + split-table dict deserializer
204
+ │ │ ├── cjson_simd.c/h # NEON / AVX2 / SSE2 kernels + runtime dispatch
205
+ │ │ ├── key_cache.h # Key/value caches & split-table dict builders
206
+ │ │ ├── numparse.h # SWAR + Clinger + Eisel-Lemire number parser
207
+ │ │ ├── parallel_ser.h # Adaptive cross-platform thread pool + atfork
208
+ │ │ ├── utf8.h # SIMD UTF-8 validation & transcoding
209
+ │ │ ├── jeaiii/ # Fast integer-to-ASCII
210
+ │ │ └── zmij/ # Fast double-to-ASCII
211
+ │ └── cxjson/ # Python package, PEP 561 stubs (__init__.pyi, py.typed)
212
+ └── tests/ # Unit, compatibility, RFC 8259 conformance/fuzz, & benchmarks
213
+ ```
214
+
215
+ ---
216
+
217
+ ## Contributing & License
218
+
219
+ Contributions are welcome—see [CONTRIBUTING.md](CONTRIBUTING.md) for build instructions, C guidelines, and how to run the fuzz/benchmark suites.
220
+
221
+ Licensed under the [MIT License](LICENSE).
222
+
@@ -0,0 +1,31 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "cxjson"
7
+ version = "0.1.0"
8
+ description = "Ultra-fast pure-C JSON library for Python with SIMD, CPython split-table dicts, and adaptive parallelism"
9
+ readme = "README.md"
10
+ license = { file = "LICENSE" }
11
+ requires-python = ">=3.9"
12
+ authors = [
13
+ { name = "James Francis" },
14
+ ]
15
+ keywords = ["json", "fast", "simd", "c-extension", "parser", "serializer", "orjson"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Programming Language :: Python :: 3.13",
22
+ "Programming Language :: Python :: 3.14",
23
+ "Programming Language :: C",
24
+ "Operating System :: OS Independent",
25
+ "Typing :: Typed",
26
+ ]
27
+
28
+ [project.urls]
29
+ Homepage = "https://github.com/jamesfrancis2004/cxjson"
30
+ Repository = "https://github.com/jamesfrancis2004/cxjson"
31
+ Issues = "https://github.com/jamesfrancis2004/cxjson/issues"
cxjson-0.1.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
cxjson-0.1.0/setup.py ADDED
@@ -0,0 +1,44 @@
1
+ import sys
2
+ from setuptools import Extension, find_packages, setup
3
+
4
+ if sys.platform == "win32":
5
+ extra_compile_args = ["/O2", "/W3"]
6
+ extra_link_args = []
7
+ else:
8
+ extra_compile_args = [
9
+ "-O3",
10
+ "-Wall",
11
+ "-Wno-unused-function",
12
+ "-flto",
13
+ "-fvisibility=hidden",
14
+ "-fomit-frame-pointer",
15
+ "-fno-math-errno",
16
+ "-fno-trapping-math",
17
+ "-fstrict-aliasing",
18
+ "-funroll-loops",
19
+ ]
20
+ extra_link_args = ["-flto"]
21
+ if sys.platform == "darwin":
22
+ extra_link_args.append("-Wl,-dead_strip")
23
+
24
+ c_extension = Extension(
25
+ name="cxjson._cxjson",
26
+ sources=[
27
+ "src/c/_cxjson_module.c",
28
+ "src/c/cjson_simd.c",
29
+ "src/c/serializer.c",
30
+ "src/c/deserializer.c",
31
+ "src/c/zmij/zmij.c",
32
+ ],
33
+ include_dirs=["src/c", "src/c/zmij", "src/c/jeaiii"],
34
+ extra_compile_args=extra_compile_args,
35
+ extra_link_args=extra_link_args,
36
+ )
37
+
38
+ setup(
39
+ packages=find_packages(where="src"),
40
+ package_dir={"": "src"},
41
+ package_data={"cxjson": ["py.typed", "__init__.pyi"]},
42
+ ext_modules=[c_extension],
43
+ )
44
+
@@ -0,0 +1,71 @@
1
+ #define PY_SSIZE_T_CLEAN
2
+ #include <Python.h>
3
+ #include "serializer.h"
4
+ #include "deserializer.h"
5
+ #include "cjson_simd.h"
6
+
7
+ static PyMethodDef CJsonMethods[] = {
8
+ {"dumps", (PyCFunction)c_json_dumps, METH_FASTCALL | METH_KEYWORDS,
9
+ "Serialize Python objects to JSON."},
10
+ {"loads", (PyCFunction)c_json_loads, METH_FASTCALL,
11
+ "Deserialize JSON to Python objects."},
12
+ {NULL, NULL, 0, NULL} /* Sentinel */
13
+ };
14
+
15
+ static struct PyModuleDef cxjsonmodule = {
16
+ PyModuleDef_HEAD_INIT,
17
+ "_cxjson",
18
+ "C extension module for cxjson",
19
+ -1,
20
+ CJsonMethods,
21
+ NULL,
22
+ NULL,
23
+ NULL,
24
+ NULL
25
+ };
26
+
27
+ PyMODINIT_FUNC PyInit__cxjson(void) {
28
+ cjson_simd_init();
29
+ if (init_serializer_types() < 0) {
30
+ return NULL;
31
+ }
32
+ if (init_deserializer_types() < 0) {
33
+ return NULL;
34
+ }
35
+
36
+ PyObject *m = PyModule_Create(&cxjsonmodule);
37
+ if (m == NULL) {
38
+ return NULL;
39
+ }
40
+
41
+ Py_INCREF(&Fragment_Type);
42
+ if (PyModule_AddObject(m, "Fragment", (PyObject *)&Fragment_Type) < 0) {
43
+ Py_DECREF(&Fragment_Type);
44
+ Py_DECREF(m);
45
+ return NULL;
46
+ }
47
+
48
+ Py_INCREF(JSONDecodeError);
49
+ if (PyModule_AddObject(m, "JSONDecodeError", JSONDecodeError) < 0) {
50
+ Py_DECREF(JSONDecodeError);
51
+ Py_DECREF(m);
52
+ return NULL;
53
+ }
54
+
55
+ PyModule_AddIntConstant(m, "OPT_INDENT_2", OPT_INDENT_2);
56
+ PyModule_AddIntConstant(m, "OPT_NAIVE_UTC", OPT_NAIVE_UTC);
57
+ PyModule_AddIntConstant(m, "OPT_NON_STR_KEYS", OPT_NON_STR_KEYS);
58
+ PyModule_AddIntConstant(m, "OPT_OMIT_MICROSECONDS", OPT_OMIT_MICROSECONDS);
59
+ PyModule_AddIntConstant(m, "OPT_SERIALIZE_NUMPY", OPT_SERIALIZE_NUMPY);
60
+ PyModule_AddIntConstant(m, "OPT_SORT_KEYS", OPT_SORT_KEYS);
61
+ PyModule_AddIntConstant(m, "OPT_STRICT_INTEGER", OPT_STRICT_INTEGER);
62
+ PyModule_AddIntConstant(m, "OPT_UTC_Z", OPT_UTC_Z);
63
+ PyModule_AddIntConstant(m, "OPT_PASSTHROUGH_SUBCLASS", OPT_PASSTHROUGH_SUBCLASS);
64
+ PyModule_AddIntConstant(m, "OPT_PASSTHROUGH_DATETIME", OPT_PASSTHROUGH_DATETIME);
65
+ PyModule_AddIntConstant(m, "OPT_APPEND_NEWLINE", OPT_APPEND_NEWLINE);
66
+ PyModule_AddIntConstant(m, "OPT_PASSTHROUGH_DATACLASS", OPT_PASSTHROUGH_DATACLASS);
67
+ PyModule_AddIntConstant(m, "OPT_SERIALIZE_DATACLASS", OPT_SERIALIZE_DATACLASS);
68
+ PyModule_AddIntConstant(m, "OPT_SERIALIZE_UUID", OPT_SERIALIZE_UUID);
69
+
70
+ return m;
71
+ }