loaderx 1.7.2__tar.gz → 1.7.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {loaderx-1.7.2/loaderx.egg-info → loaderx-1.7.7}/PKG-INFO +85 -50
  2. {loaderx-1.7.2 → loaderx-1.7.7}/README.md +84 -49
  3. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx/__init__.py +1 -1
  4. {loaderx-1.7.2 → loaderx-1.7.7/loaderx.egg-info}/PKG-INFO +85 -50
  5. {loaderx-1.7.2 → loaderx-1.7.7}/scripts/_bench_common.py +219 -87
  6. loaderx-1.7.7/scripts/bench_ragged.py +632 -0
  7. loaderx-1.7.2/scripts/bench_ragged.py +0 -375
  8. {loaderx-1.7.2 → loaderx-1.7.7}/LICENSE +0 -0
  9. {loaderx-1.7.2 → loaderx-1.7.7}/MANIFEST.in +0 -0
  10. {loaderx-1.7.2 → loaderx-1.7.7}/build.zig +0 -0
  11. {loaderx-1.7.2 → loaderx-1.7.7}/build.zig.zon +0 -0
  12. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx/_lib.py +0 -0
  13. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx/_store.py +0 -0
  14. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx/dataloader.py +0 -0
  15. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx/utils.py +0 -0
  16. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx/zrecord.py +0 -0
  17. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx/zsampler.py +0 -0
  18. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx.egg-info/SOURCES.txt +0 -0
  19. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx.egg-info/dependency_links.txt +0 -0
  20. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx.egg-info/requires.txt +0 -0
  21. {loaderx-1.7.2 → loaderx-1.7.7}/loaderx.egg-info/top_level.txt +0 -0
  22. {loaderx-1.7.2 → loaderx-1.7.7}/pyproject.toml +0 -0
  23. {loaderx-1.7.2 → loaderx-1.7.7}/scripts/bench.py +0 -0
  24. {loaderx-1.7.2 → loaderx-1.7.7}/scripts/bench_dense.py +0 -0
  25. {loaderx-1.7.2 → loaderx-1.7.7}/scripts/build_wheels.py +0 -0
  26. {loaderx-1.7.2 → loaderx-1.7.7}/scripts/test_loaderx.py +0 -0
  27. {loaderx-1.7.2 → loaderx-1.7.7}/setup.cfg +0 -0
  28. {loaderx-1.7.2 → loaderx-1.7.7}/setup.py +0 -0
  29. {loaderx-1.7.2 → loaderx-1.7.7}/src/record/codec.zig +0 -0
  30. {loaderx-1.7.2 → loaderx-1.7.7}/src/record/engine.zig +0 -0
  31. {loaderx-1.7.2 → loaderx-1.7.7}/src/record/exec.zig +0 -0
  32. {loaderx-1.7.2 → loaderx-1.7.7}/src/record/format.zig +0 -0
  33. {loaderx-1.7.2 → loaderx-1.7.7}/src/record/storage.zig +0 -0
  34. {loaderx-1.7.2 → loaderx-1.7.7}/src/store/common.zig +0 -0
  35. {loaderx-1.7.2 → loaderx-1.7.7}/src/store/dense.zig +0 -0
  36. {loaderx-1.7.2 → loaderx-1.7.7}/src/store/ragged.zig +0 -0
  37. {loaderx-1.7.2 → loaderx-1.7.7}/src/store.zig +0 -0
  38. {loaderx-1.7.2 → loaderx-1.7.7}/src/zsampler.zig +0 -0
  39. {loaderx-1.7.2 → loaderx-1.7.7}/src/zstd/c.zig +0 -0
  40. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/COPYING +0 -0
  41. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/LICENSE +0 -0
  42. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/allocations.h +0 -0
  43. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/bits.h +0 -0
  44. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/bitstream.h +0 -0
  45. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/compiler.h +0 -0
  46. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/cpu.h +0 -0
  47. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/debug.c +0 -0
  48. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/debug.h +0 -0
  49. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/entropy_common.c +0 -0
  50. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/error_private.c +0 -0
  51. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/error_private.h +0 -0
  52. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/fse.h +0 -0
  53. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/fse_decompress.c +0 -0
  54. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/huf.h +0 -0
  55. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/mem.h +0 -0
  56. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/pool.c +0 -0
  57. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/pool.h +0 -0
  58. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/portability_macros.h +0 -0
  59. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/threading.c +0 -0
  60. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/threading.h +0 -0
  61. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/xxhash.c +0 -0
  62. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/xxhash.h +0 -0
  63. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/zstd_common.c +0 -0
  64. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/zstd_deps.h +0 -0
  65. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/zstd_internal.h +0 -0
  66. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/common/zstd_trace.h +0 -0
  67. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/clevels.h +0 -0
  68. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/fse_compress.c +0 -0
  69. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/hist.c +0 -0
  70. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/hist.h +0 -0
  71. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/huf_compress.c +0 -0
  72. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress.c +0 -0
  73. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress_internal.h +0 -0
  74. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress_literals.c +0 -0
  75. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress_literals.h +0 -0
  76. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress_sequences.c +0 -0
  77. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress_sequences.h +0 -0
  78. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress_superblock.c +0 -0
  79. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_compress_superblock.h +0 -0
  80. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_cwksp.h +0 -0
  81. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_double_fast.c +0 -0
  82. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_double_fast.h +0 -0
  83. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_fast.c +0 -0
  84. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_fast.h +0 -0
  85. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_lazy.c +0 -0
  86. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_lazy.h +0 -0
  87. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_ldm.c +0 -0
  88. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_ldm.h +0 -0
  89. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_ldm_geartab.h +0 -0
  90. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_opt.c +0 -0
  91. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstd_opt.h +0 -0
  92. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstdmt_compress.c +0 -0
  93. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/compress/zstdmt_compress.h +0 -0
  94. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/decompress/huf_decompress.c +0 -0
  95. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/decompress/zstd_ddict.c +0 -0
  96. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/decompress/zstd_ddict.h +0 -0
  97. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/decompress/zstd_decompress.c +0 -0
  98. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/decompress/zstd_decompress_block.c +0 -0
  99. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/decompress/zstd_decompress_block.h +0 -0
  100. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/decompress/zstd_decompress_internal.h +0 -0
  101. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/dictBuilder/cover.c +0 -0
  102. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/dictBuilder/cover.h +0 -0
  103. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/dictBuilder/divsufsort.c +0 -0
  104. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/dictBuilder/divsufsort.h +0 -0
  105. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/dictBuilder/fastcover.c +0 -0
  106. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/dictBuilder/zdict.c +0 -0
  107. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/zdict.h +0 -0
  108. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/zstd.h +0 -0
  109. {loaderx-1.7.2 → loaderx-1.7.7}/vendor/zstd/lib/zstd_errors.h +0 -0
{loaderx-1.7.2/loaderx.egg-info → loaderx-1.7.7}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: loaderx
3
- Version: 1.7.2
3
+ Version: 1.7.7
4
4
  Summary: Zrecord typed persistent stores and the Loaderx data loader for single-machine AI pipelines
5
5
  Author-email: Ben0i0d <ben0i0d@foxmail.com>
6
6
  License: MIT License
@@ -74,6 +74,11 @@ loaderx is built around several core principles:
74
74
  list. Neither is padded, and equal length is never treated as a special case
75
75
  of variable length.
76
76
 
77
+ ## 设计文档
78
+
79
+ - [性能优化与工程决策记录](docs/性能优化.md)
80
+ - [分布式架构草案](docs/分布式架构.md)
81
+
77
82
  ## Quick Start
78
83
  ```python
79
84
  from loaderx.zrecord import DenseStore
@@ -397,19 +402,25 @@ For practical integration examples, please refer to the **[Data2Latent](https://
397
402
 
398
403
  Dense and Ragged are measured separately because they expose different contracts
399
404
  and have different useful metrics. `scripts/bench_dense.py` measures fixed-shape
400
- random gather, `scripts/bench_ragged.py` measures variable-length records, and
405
+ random gather, `scripts/bench_ragged.py` measures variable-shape records, and
401
406
  `scripts/bench.py` covers machine, sampler, and the original dense loader
402
407
  comparison. Every path runs through the public Python binding, so CFFI, NumPy
403
408
  allocation, and Ragged list/shape reconstruction are inside the timings.
404
409
 
405
410
  ### Methodology
406
411
 
407
- These `1.7.2` store results are one qualitative pass on a warm page cache. Each
408
- backend receives identical source records and random index plans. Before timing,
409
- the benchmark sweeps every record and validates every planned result for exact
410
- dtype, shape, order, and values. `write` is page-cache ingestion with durability
411
- deferred consistently across backends; call `sync()` explicitly when measuring
412
- Zrecord durability. Disk size is allocated blocks, not sparse apparent size.
412
+ The `1.7.7` Dense and Ragged snapshots below are each one qualitative pass on a
413
+ warm page cache. Within each benchmark, every backend receives
414
+ identical source records and random index plans. Before timing, the benchmark
415
+ sweeps every record and validates every planned result for exact
416
+ dtype, shape, order, and values. `logical write` and `logical gather` divide
417
+ uncompressed NumPy payload bytes by elapsed time; they measure bytes accepted or
418
+ returned by the public API, not physical storage bandwidth. Write is page-cache
419
+ ingestion with durability deferred consistently across backends; call `sync()`
420
+ explicitly when measuring Zrecord durability. Disk size is allocated blocks,
421
+ not sparse apparent size.
422
+ Every backend name states its actual codec; the full default set is required
423
+ rather than silently skipped when a package is missing.
413
424
 
414
425
  **Machine** — one local workstation (AMD Ryzen AI 9 HX PRO 370, 12 cores / 24
415
426
  threads):
@@ -453,46 +464,67 @@ comparison; other record sizes are a `--shape` away.
453
464
 
454
465
  Realistic records — 256 KiB per record, 64 MiB per batch:
455
466
 
456
- | store | write | gather | on disk | ratio |
457
- |-------------------|------------|-------------|---------|-------|
458
- | zrecord-zstd | 911 MiB/s | 3987 MiB/s | 555 MiB | 1.13x |
459
- | zrecord-zstdict | 39 MiB/s | 1531 MiB/s | 388 MiB | 1.61x |
460
- | zrecord-raw | 2531 MiB/s | 7894 MiB/s | 625 MiB | 1.00x |
461
- | npy-mmap | 2403 MiB/s | 3418 MiB/s | 625 MiB | 1.00x |
462
- | hdf5 | 2517 MiB/s | 1871 MiB/s | 625 MiB | 1.00x |
463
- | blosc2 | 63 MiB/s | 543 MiB/s | 536 MiB | 1.17x |
464
- | tensorstore | 345 MiB/s | 268 MiB/s | 462 MiB | 1.35x |
467
+ Dense snapshot: `1.7.7`.
468
+
469
+ | store | logical write | logical gather | on disk | ratio |
470
+ |----------------------------|--------------:|---------------:|--------:|------:|
471
+ | zrecord-zstd | 1073 MiB/s | 4572 MiB/s | 555 MiB | 1.13x |
472
+ | zrecord-zstdict | 48 MiB/s | 2051 MiB/s | 388 MiB | 1.61x |
473
+ | zrecord-raw | 2325 MiB/s | 7996 MiB/s | 625 MiB | 1.00x |
474
+ | npy-mmap-raw | 2824 MiB/s | 3880 MiB/s | 625 MiB | 1.00x |
475
+ | hdf5-raw | 2385 MiB/s | 2155 MiB/s | 625 MiB | 1.00x |
476
+ | hdf5-gzip | 50 MiB/s | 234 MiB/s | 445 MiB | 1.40x |
477
+ | lmdb-raw | 794 MiB/s | 3406 MiB/s | 635 MiB | 0.98x |
478
+ | arrow-ipc-raw | 1657 MiB/s | 2601 MiB/s | 625 MiB | 1.00x |
479
+ | arrow-ipc-zstd | 311 MiB/s | 104 MiB/s | 418 MiB | 1.49x |
480
+ | parquet-raw | 1220 MiB/s | 442 MiB/s | 625 MiB | 1.00x |
481
+ | parquet-zstd | 311 MiB/s | 91 MiB/s | 418 MiB | 1.49x |
482
+ | arrayrecord-raw | 1878 MiB/s | 1873 MiB/s | 625 MiB | 1.00x |
483
+ | arrayrecord-zstd | 481 MiB/s | 1530 MiB/s | 567 MiB | 1.10x |
484
+ | tiledb-raw | 818 MiB/s | 780 MiB/s | 625 MiB | 1.00x |
485
+ | tiledb-zstd | 812 MiB/s | 811 MiB/s | 553 MiB | 1.13x |
465
486
 
466
487
  At 256 KiB per record, a 64 MiB batch exceeds L3 and the raw paths are primarily
467
- DRAM-bandwidth-bound. Zrecord-raw reaches 7.7 GiB/s and is 2.3x npy-mmap here;
468
- plain zstd gathers at 3.9 GiB/s. DenseStore demonstrates that typed record
469
- ownership and per-record compression do not turn fixed tensors into an object-
470
- store slow path.
471
-
472
- ### RaggedStore: Variable-Length Records
473
-
474
- This workload contains 100,000 int32 token sequences with lognormal lengths
475
- clipped to 16..2048 (observed median 253), totaling 142.7 MiB of logical payload.
476
- Each of 100 random batches contains 256 records. Every backend must return an
477
- ordered `list[np.ndarray]`; returning only flat values and offsets is not enough.
478
-
479
- | backend | write | gather | krecords/s | p95 | disk | ratio |
480
- |-------------------|------:|-------:|-----------:|----:|-----:|------:|
481
- | zrecord-zstd | 127 MiB/s | 317 MiB/s | 221.0 | 1.48 ms | 28.1 MiB | 5.08x |
482
- | zrecord-zstdict | 23 MiB/s | 314 MiB/s | 219.0 | 1.48 ms | 17.1 MiB | 8.32x |
483
- | zrecord-raw | 499 MiB/s | 313 MiB/s | 218.2 | 1.72 ms | 147.2 MiB | 0.97x |
484
- | packed-offsets | 1507 MiB/s | 976 MiB/s | 681.3 | 0.53 ms | 143.4 MiB | 0.99x |
485
- | hdf5-vlen | 36 MiB/s | 27 MiB/s | 18.9 | 20.01 ms | 146.4 MiB | 0.97x |
486
-
487
- `packed-offsets` is the useful raw lower bound: a specialized payload file plus
488
- one offsets array for one-dimensional tokens. It is not a feature-equivalent
489
- store: it has no compression, arbitrary-rank shape records, native typed owner,
490
- paired append publication, swap-delete, or compact. Zrecord pays for those
491
- semantics, while still gathering about 11.6x as many records/s as HDF5 vlen. The
492
- compressed paths gather at essentially raw Zrecord speed for this small-record
493
- workload; `zstdict` reduces 142.7 MiB to 17.1 MiB (8.32x) at 1.48 ms p95. Disk
488
+ DRAM-bandwidth-bound. Zrecord-raw reaches 7.8 GiB/s and is 2.1x npy-mmap-raw here;
489
+ plain zstd gathers at 4.5 GiB/s. LMDB and Arrow IPC are competitive raw record
490
+ stores, while codecs tied to whole IPC batches or Parquet row groups pay read
491
+ amplification on random gathers. DenseStore demonstrates that
492
+ typed record ownership and per-record compression do not turn fixed tensors
493
+ into an object-store slow path.
494
+
495
+ ### RaggedStore: Variable-Shape Records
496
+
497
+ This workload contains 2,500 variable-resolution CHW RGB images. Height and
498
+ width are independently lognormal and clipped to 64..512 (observed medians
499
+ 223 and 224), totaling 417.1 MiB of logical uint8 payload. Each of 50 random
500
+ batches contains 256 records. Every backend persists payload plus exact shape
501
+ and must return an ordered `list[np.ndarray]` of `(3, H, W)` arrays; a flat byte
502
+ list or a one-dimensional variable-length abstraction is not enough.
503
+
504
+ | backend | logical write | logical gather | krecords/s | p95 | disk | ratio |
505
+ |----------------------------|--------------:|---------------:|-----------:|----:|-----:|------:|
506
+ | zrecord-zstd | 2095 MiB/s | 5738 MiB/s | 34.6 | 8.17 ms | 27.1 MiB | 15.42x |
507
+ | zrecord-zstdict | 38 MiB/s | 6271 MiB/s | 37.8 | 7.64 ms | 17.8 MiB | 23.38x |
508
+ | zrecord-raw | 1493 MiB/s | 7017 MiB/s | 42.3 | 7.16 ms | 417.3 MiB | 1.00x |
509
+ | hdf5-raw | 1055 MiB/s | 1258 MiB/s | 7.6 | 40.55 ms | 418.0 MiB | 1.00x |
510
+ | hdf5-gzip | 243 MiB/s | 119 MiB/s | 0.7 | 406.67 ms | 30.0 MiB | 13.92x |
511
+ | lmdb-raw | 914 MiB/s | 6965 MiB/s | 42.0 | 6.63 ms | 422.2 MiB | 0.99x |
512
+ | arrow-ipc-raw | 1448 MiB/s | 3950 MiB/s | 23.8 | 13.03 ms | 417.2 MiB | 1.00x |
513
+ | arrow-ipc-zstd | 529 MiB/s | 181 MiB/s | 1.1 | 268.28 ms | 26.0 MiB | 16.02x |
514
+ | parquet-raw | 977 MiB/s | 466 MiB/s | 2.8 | 100.94 ms | 417.2 MiB | 1.00x |
515
+ | parquet-zstd | 510 MiB/s | 165 MiB/s | 1.0 | 288.08 ms | 26.0 MiB | 16.03x |
516
+ | arrayrecord-raw | 1417 MiB/s | 2612 MiB/s | 15.8 | 18.69 ms | 417.6 MiB | 1.00x |
517
+ | arrayrecord-zstd | 861 MiB/s | 1449 MiB/s | 8.7 | 39.32 ms | 27.3 MiB | 15.27x |
518
+ | tiledb-raw | 419 MiB/s | 50 MiB/s | 0.3 | 919.16 ms | 417.2 MiB | 1.00x |
519
+ | tiledb-zstd | 618 MiB/s | 124 MiB/s | 0.8 | 367.07 ms | 27.1 MiB | 15.40x |
520
+
521
+ LMDB is the fastest raw record contender and Arrow IPC is the strongest raw
522
+ typed-file alternative. Zrecord-zstd delivers 5.6 GiB/s of logical payload while
523
+ reducing the corpus to 27.1 MiB. HDF5, Arrow IPC, Parquet, ArrayRecord and TileDB
524
+ show the same framework/codec tradeoffs in both tables; compressed batch, chunk
525
+ and row-group formats pay read amplification on random records. Disk
494
526
  includes shape records, both loc entries, codec frames, dictionaries, and global
495
- headers. Arrow IPC was unavailable on this machine and was skipped.
527
+ headers where the format uses them.
496
528
 
497
529
  ### DataLoader: Fixed-Shape Pipeline
498
530
 
@@ -529,7 +561,9 @@ and `--transform augment`); they are not part of the default `all`:
529
561
  * **Sampler** — zsampler draws ~1.9–2x faster than free-threaded numpy (the
530
562
  256-row run was a noisy 2.46x), the same broad margin as on the GIL build: a
531
563
  batch is one Zig call either way.
532
- * **Store** — the free-threaded numbers track the GIL table closely.
564
+ * **Historical Store snapshot** — these older rows predate the symmetric
565
+ benchmark matrix and are retained only for the GIL/free-threaded comparison.
566
+ The free-threaded numbers track the GIL table closely.
533
567
  `zrecord-raw` gathers 8.9 vs 8.8 GiB/s and `zrecord-zstd` 4.7 vs 4.6 GiB/s.
534
568
  The no-GIL table was run without blosc2 so its import could not change the
535
569
  process mode. blosc2 was measured separately at 678 MiB/s; its extension
@@ -540,10 +574,10 @@ and `--transform augment`); they are not part of the default `all`:
540
574
  |--------------|--------------------|---------------------|
541
575
  | zrecord-raw | 8803 MiB/s | 8878 MiB/s |
542
576
  | zrecord-zstd | 4603 MiB/s | 4707 MiB/s |
543
- | npy-mmap | 4408 MiB/s | 3974 MiB/s |
544
- | hdf5 | 2288 MiB/s | 2210 MiB/s |
545
- | blosc2 | 646 MiB/s | 678 MiB/s (GIL enabled) |
546
- | tensorstore | 299 MiB/s | 323 MiB/s |
577
+ | npy-mmap-raw | 4408 MiB/s | 3974 MiB/s |
578
+ | hdf5-raw | 2288 MiB/s | 2210 MiB/s |
579
+ | blosc2-zstd | 646 MiB/s | 678 MiB/s (GIL enabled) |
580
+ | tensorstore-zarr3-gzip | 299 MiB/s | 323 MiB/s |
547
581
  * **Loader, identity** — unchanged on both interpreters at 64 MiB per batch:
548
582
 
549
583
  | loader | CPython 3.14 (GIL) | free-threaded 3.14t |
@@ -685,6 +719,7 @@ explicit memory bound while zrecord independently caps filesystem writes.
685
719
  zig build # host shared objects, into loaderx/lib/
686
720
  zig build test # native store suite, in both Debug and ReleaseFast
687
721
  python3 scripts/test_loaderx.py # Python integration suite against the real build
722
+ uv pip install -r scripts/requirements-bench.txt
688
723
  python3 scripts/bench_dense.py # fixed-shape store comparison
689
724
  python3 scripts/bench_ragged.py # variable-length store comparison
690
725
  python3 scripts/bench.py # machine, sampler, and dense loader layers
{loaderx-1.7.2 → loaderx-1.7.7}/README.md RENAMED
@@ -27,6 +27,11 @@ loaderx is built around several core principles:
27
27
  list. Neither is padded, and equal length is never treated as a special case
28
28
  of variable length.
29
29
 
30
+ ## 设计文档
31
+
32
+ - [性能优化与工程决策记录](docs/性能优化.md)
33
+ - [分布式架构草案](docs/分布式架构.md)
34
+
30
35
  ## Quick Start
31
36
  ```python
32
37
  from loaderx.zrecord import DenseStore
@@ -350,19 +355,25 @@ For practical integration examples, please refer to the **[Data2Latent](https://
350
355
 
351
356
  Dense and Ragged are measured separately because they expose different contracts
352
357
  and have different useful metrics. `scripts/bench_dense.py` measures fixed-shape
353
- random gather, `scripts/bench_ragged.py` measures variable-length records, and
358
+ random gather, `scripts/bench_ragged.py` measures variable-shape records, and
354
359
  `scripts/bench.py` covers machine, sampler, and the original dense loader
355
360
  comparison. Every path runs through the public Python binding, so CFFI, NumPy
356
361
  allocation, and Ragged list/shape reconstruction are inside the timings.
357
362
 
358
363
  ### Methodology
359
364
 
360
- These `1.7.2` store results are one qualitative pass on a warm page cache. Each
361
- backend receives identical source records and random index plans. Before timing,
362
- the benchmark sweeps every record and validates every planned result for exact
363
- dtype, shape, order, and values. `write` is page-cache ingestion with durability
364
- deferred consistently across backends; call `sync()` explicitly when measuring
365
- Zrecord durability. Disk size is allocated blocks, not sparse apparent size.
365
+ The `1.7.7` Dense and Ragged snapshots below are each one qualitative pass on a
366
+ warm page cache. Within each benchmark, every backend receives
367
+ identical source records and random index plans. Before timing, the benchmark
368
+ sweeps every record and validates every planned result for exact
369
+ dtype, shape, order, and values. `logical write` and `logical gather` divide
370
+ uncompressed NumPy payload bytes by elapsed time; they measure bytes accepted or
371
+ returned by the public API, not physical storage bandwidth. Write is page-cache
372
+ ingestion with durability deferred consistently across backends; call `sync()`
373
+ explicitly when measuring Zrecord durability. Disk size is allocated blocks,
374
+ not sparse apparent size.
375
+ Every backend name states its actual codec; the full default set is required
376
+ rather than silently skipped when a package is missing.
366
377
 
367
378
  **Machine** — one local workstation (AMD Ryzen AI 9 HX PRO 370, 12 cores / 24
368
379
  threads):
@@ -406,46 +417,67 @@ comparison; other record sizes are a `--shape` away.
406
417
 
407
418
  Realistic records — 256 KiB per record, 64 MiB per batch:
408
419
 
409
- | store | write | gather | on disk | ratio |
410
- |-------------------|------------|-------------|---------|-------|
411
- | zrecord-zstd | 911 MiB/s | 3987 MiB/s | 555 MiB | 1.13x |
412
- | zrecord-zstdict | 39 MiB/s | 1531 MiB/s | 388 MiB | 1.61x |
413
- | zrecord-raw | 2531 MiB/s | 7894 MiB/s | 625 MiB | 1.00x |
414
- | npy-mmap | 2403 MiB/s | 3418 MiB/s | 625 MiB | 1.00x |
415
- | hdf5 | 2517 MiB/s | 1871 MiB/s | 625 MiB | 1.00x |
416
- | blosc2 | 63 MiB/s | 543 MiB/s | 536 MiB | 1.17x |
417
- | tensorstore | 345 MiB/s | 268 MiB/s | 462 MiB | 1.35x |
420
+ Dense snapshot: `1.7.7`.
421
+
422
+ | store | logical write | logical gather | on disk | ratio |
423
+ |----------------------------|--------------:|---------------:|--------:|------:|
424
+ | zrecord-zstd | 1073 MiB/s | 4572 MiB/s | 555 MiB | 1.13x |
425
+ | zrecord-zstdict | 48 MiB/s | 2051 MiB/s | 388 MiB | 1.61x |
426
+ | zrecord-raw | 2325 MiB/s | 7996 MiB/s | 625 MiB | 1.00x |
427
+ | npy-mmap-raw | 2824 MiB/s | 3880 MiB/s | 625 MiB | 1.00x |
428
+ | hdf5-raw | 2385 MiB/s | 2155 MiB/s | 625 MiB | 1.00x |
429
+ | hdf5-gzip | 50 MiB/s | 234 MiB/s | 445 MiB | 1.40x |
430
+ | lmdb-raw | 794 MiB/s | 3406 MiB/s | 635 MiB | 0.98x |
431
+ | arrow-ipc-raw | 1657 MiB/s | 2601 MiB/s | 625 MiB | 1.00x |
432
+ | arrow-ipc-zstd | 311 MiB/s | 104 MiB/s | 418 MiB | 1.49x |
433
+ | parquet-raw | 1220 MiB/s | 442 MiB/s | 625 MiB | 1.00x |
434
+ | parquet-zstd | 311 MiB/s | 91 MiB/s | 418 MiB | 1.49x |
435
+ | arrayrecord-raw | 1878 MiB/s | 1873 MiB/s | 625 MiB | 1.00x |
436
+ | arrayrecord-zstd | 481 MiB/s | 1530 MiB/s | 567 MiB | 1.10x |
437
+ | tiledb-raw | 818 MiB/s | 780 MiB/s | 625 MiB | 1.00x |
438
+ | tiledb-zstd | 812 MiB/s | 811 MiB/s | 553 MiB | 1.13x |
418
439
 
419
440
  At 256 KiB per record, a 64 MiB batch exceeds L3 and the raw paths are primarily
420
- DRAM-bandwidth-bound. Zrecord-raw reaches 7.7 GiB/s and is 2.3x npy-mmap here;
421
- plain zstd gathers at 3.9 GiB/s. DenseStore demonstrates that typed record
422
- ownership and per-record compression do not turn fixed tensors into an object-
423
- store slow path.
424
-
425
- ### RaggedStore: Variable-Length Records
426
-
427
- This workload contains 100,000 int32 token sequences with lognormal lengths
428
- clipped to 16..2048 (observed median 253), totaling 142.7 MiB of logical payload.
429
- Each of 100 random batches contains 256 records. Every backend must return an
430
- ordered `list[np.ndarray]`; returning only flat values and offsets is not enough.
431
-
432
- | backend | write | gather | krecords/s | p95 | disk | ratio |
433
- |-------------------|------:|-------:|-----------:|----:|-----:|------:|
434
- | zrecord-zstd | 127 MiB/s | 317 MiB/s | 221.0 | 1.48 ms | 28.1 MiB | 5.08x |
435
- | zrecord-zstdict | 23 MiB/s | 314 MiB/s | 219.0 | 1.48 ms | 17.1 MiB | 8.32x |
436
- | zrecord-raw | 499 MiB/s | 313 MiB/s | 218.2 | 1.72 ms | 147.2 MiB | 0.97x |
437
- | packed-offsets | 1507 MiB/s | 976 MiB/s | 681.3 | 0.53 ms | 143.4 MiB | 0.99x |
438
- | hdf5-vlen | 36 MiB/s | 27 MiB/s | 18.9 | 20.01 ms | 146.4 MiB | 0.97x |
439
-
440
- `packed-offsets` is the useful raw lower bound: a specialized payload file plus
441
- one offsets array for one-dimensional tokens. It is not a feature-equivalent
442
- store: it has no compression, arbitrary-rank shape records, native typed owner,
443
- paired append publication, swap-delete, or compact. Zrecord pays for those
444
- semantics, while still gathering about 11.6x as many records/s as HDF5 vlen. The
445
- compressed paths gather at essentially raw Zrecord speed for this small-record
446
- workload; `zstdict` reduces 142.7 MiB to 17.1 MiB (8.32x) at 1.48 ms p95. Disk
441
+ DRAM-bandwidth-bound. Zrecord-raw reaches 7.8 GiB/s and is 2.1x npy-mmap-raw here;
442
+ plain zstd gathers at 4.5 GiB/s. LMDB and Arrow IPC are competitive raw record
443
+ stores, while codecs tied to whole IPC batches or Parquet row groups pay read
444
+ amplification on random gathers. DenseStore demonstrates that
445
+ typed record ownership and per-record compression do not turn fixed tensors
446
+ into an object-store slow path.
447
+
448
+ ### RaggedStore: Variable-Shape Records
449
+
450
+ This workload contains 2,500 variable-resolution CHW RGB images. Height and
451
+ width are independently lognormal and clipped to 64..512 (observed medians
452
+ 223 and 224), totaling 417.1 MiB of logical uint8 payload. Each of 50 random
453
+ batches contains 256 records. Every backend persists payload plus exact shape
454
+ and must return an ordered `list[np.ndarray]` of `(3, H, W)` arrays; a flat byte
455
+ list or a one-dimensional variable-length abstraction is not enough.
456
+
457
+ | backend | logical write | logical gather | krecords/s | p95 | disk | ratio |
458
+ |----------------------------|--------------:|---------------:|-----------:|----:|-----:|------:|
459
+ | zrecord-zstd | 2095 MiB/s | 5738 MiB/s | 34.6 | 8.17 ms | 27.1 MiB | 15.42x |
460
+ | zrecord-zstdict | 38 MiB/s | 6271 MiB/s | 37.8 | 7.64 ms | 17.8 MiB | 23.38x |
461
+ | zrecord-raw | 1493 MiB/s | 7017 MiB/s | 42.3 | 7.16 ms | 417.3 MiB | 1.00x |
462
+ | hdf5-raw | 1055 MiB/s | 1258 MiB/s | 7.6 | 40.55 ms | 418.0 MiB | 1.00x |
463
+ | hdf5-gzip | 243 MiB/s | 119 MiB/s | 0.7 | 406.67 ms | 30.0 MiB | 13.92x |
464
+ | lmdb-raw | 914 MiB/s | 6965 MiB/s | 42.0 | 6.63 ms | 422.2 MiB | 0.99x |
465
+ | arrow-ipc-raw | 1448 MiB/s | 3950 MiB/s | 23.8 | 13.03 ms | 417.2 MiB | 1.00x |
466
+ | arrow-ipc-zstd | 529 MiB/s | 181 MiB/s | 1.1 | 268.28 ms | 26.0 MiB | 16.02x |
467
+ | parquet-raw | 977 MiB/s | 466 MiB/s | 2.8 | 100.94 ms | 417.2 MiB | 1.00x |
468
+ | parquet-zstd | 510 MiB/s | 165 MiB/s | 1.0 | 288.08 ms | 26.0 MiB | 16.03x |
469
+ | arrayrecord-raw | 1417 MiB/s | 2612 MiB/s | 15.8 | 18.69 ms | 417.6 MiB | 1.00x |
470
+ | arrayrecord-zstd | 861 MiB/s | 1449 MiB/s | 8.7 | 39.32 ms | 27.3 MiB | 15.27x |
471
+ | tiledb-raw | 419 MiB/s | 50 MiB/s | 0.3 | 919.16 ms | 417.2 MiB | 1.00x |
472
+ | tiledb-zstd | 618 MiB/s | 124 MiB/s | 0.8 | 367.07 ms | 27.1 MiB | 15.40x |
473
+
474
+ LMDB is the fastest raw record contender and Arrow IPC is the strongest raw
475
+ typed-file alternative. Zrecord-zstd delivers 5.6 GiB/s of logical payload while
476
+ reducing the corpus to 27.1 MiB. HDF5, Arrow IPC, Parquet, ArrayRecord and TileDB
477
+ show the same framework/codec tradeoffs in both tables; compressed batch, chunk
478
+ and row-group formats pay read amplification on random records. Disk
447
479
  includes shape records, both loc entries, codec frames, dictionaries, and global
448
- headers. Arrow IPC was unavailable on this machine and was skipped.
480
+ headers where the format uses them.
449
481
 
450
482
  ### DataLoader: Fixed-Shape Pipeline
451
483
 
@@ -482,7 +514,9 @@ and `--transform augment`); they are not part of the default `all`:
482
514
  * **Sampler** — zsampler draws ~1.9–2x faster than free-threaded numpy (the
483
515
  256-row run was a noisy 2.46x), the same broad margin as on the GIL build: a
484
516
  batch is one Zig call either way.
485
- * **Store** — the free-threaded numbers track the GIL table closely.
517
+ * **Historical Store snapshot** — these older rows predate the symmetric
518
+ benchmark matrix and are retained only for the GIL/free-threaded comparison.
519
+ The free-threaded numbers track the GIL table closely.
486
520
  `zrecord-raw` gathers 8.9 vs 8.8 GiB/s and `zrecord-zstd` 4.7 vs 4.6 GiB/s.
487
521
  The no-GIL table was run without blosc2 so its import could not change the
488
522
  process mode. blosc2 was measured separately at 678 MiB/s; its extension
@@ -493,10 +527,10 @@ and `--transform augment`); they are not part of the default `all`:
493
527
  |--------------|--------------------|---------------------|
494
528
  | zrecord-raw | 8803 MiB/s | 8878 MiB/s |
495
529
  | zrecord-zstd | 4603 MiB/s | 4707 MiB/s |
496
- | npy-mmap | 4408 MiB/s | 3974 MiB/s |
497
- | hdf5 | 2288 MiB/s | 2210 MiB/s |
498
- | blosc2 | 646 MiB/s | 678 MiB/s (GIL enabled) |
499
- | tensorstore | 299 MiB/s | 323 MiB/s |
530
+ | npy-mmap-raw | 4408 MiB/s | 3974 MiB/s |
531
+ | hdf5-raw | 2288 MiB/s | 2210 MiB/s |
532
+ | blosc2-zstd | 646 MiB/s | 678 MiB/s (GIL enabled) |
533
+ | tensorstore-zarr3-gzip | 299 MiB/s | 323 MiB/s |
500
534
  * **Loader, identity** — unchanged on both interpreters at 64 MiB per batch:
501
535
 
502
536
  | loader | CPython 3.14 (GIL) | free-threaded 3.14t |
@@ -638,6 +672,7 @@ explicit memory bound while zrecord independently caps filesystem writes.
638
672
  zig build # host shared objects, into loaderx/lib/
639
673
  zig build test # native store suite, in both Debug and ReleaseFast
640
674
  python3 scripts/test_loaderx.py # Python integration suite against the real build
675
+ uv pip install -r scripts/requirements-bench.txt
641
676
  python3 scripts/bench_dense.py # fixed-shape store comparison
642
677
  python3 scripts/bench_ragged.py # variable-length store comparison
643
678
  python3 scripts/bench.py # machine, sampler, and dense loader layers
{loaderx-1.7.2 → loaderx-1.7.7}/loaderx/__init__.py RENAMED
@@ -16,4 +16,4 @@ pass-throughs; ``dataloader`` dynamically composes a dict of named streams and
16
16
  produces the endless, step-based sampler + prefetch pipeline over them.
17
17
  """
18
18
 
19
- __version__ = "1.7.2"
19
+ __version__ = "1.7.7"
{loaderx-1.7.2 → loaderx-1.7.7/loaderx.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: loaderx
3
- Version: 1.7.2
3
+ Version: 1.7.7
4
4
  Summary: Zrecord typed persistent stores and the Loaderx data loader for single-machine AI pipelines
5
5
  Author-email: Ben0i0d <ben0i0d@foxmail.com>
6
6
  License: MIT License
@@ -74,6 +74,11 @@ loaderx is built around several core principles:
74
74
  list. Neither is padded, and equal length is never treated as a special case
75
75
  of variable length.
76
76
 
77
+ ## 设计文档
78
+
79
+ - [性能优化与工程决策记录](docs/性能优化.md)
80
+ - [分布式架构草案](docs/分布式架构.md)
81
+
77
82
  ## Quick Start
78
83
  ```python
79
84
  from loaderx.zrecord import DenseStore
@@ -397,19 +402,25 @@ For practical integration examples, please refer to the **[Data2Latent](https://
397
402
 
398
403
  Dense and Ragged are measured separately because they expose different contracts
399
404
  and have different useful metrics. `scripts/bench_dense.py` measures fixed-shape
400
- random gather, `scripts/bench_ragged.py` measures variable-length records, and
405
+ random gather, `scripts/bench_ragged.py` measures variable-shape records, and
401
406
  `scripts/bench.py` covers machine, sampler, and the original dense loader
402
407
  comparison. Every path runs through the public Python binding, so CFFI, NumPy
403
408
  allocation, and Ragged list/shape reconstruction are inside the timings.
404
409
 
405
410
  ### Methodology
406
411
 
407
- These `1.7.2` store results are one qualitative pass on a warm page cache. Each
408
- backend receives identical source records and random index plans. Before timing,
409
- the benchmark sweeps every record and validates every planned result for exact
410
- dtype, shape, order, and values. `write` is page-cache ingestion with durability
411
- deferred consistently across backends; call `sync()` explicitly when measuring
412
- Zrecord durability. Disk size is allocated blocks, not sparse apparent size.
412
+ The `1.7.7` Dense and Ragged snapshots below are each one qualitative pass on a
413
+ warm page cache. Within each benchmark, every backend receives
414
+ identical source records and random index plans. Before timing, the benchmark
415
+ sweeps every record and validates every planned result for exact
416
+ dtype, shape, order, and values. `logical write` and `logical gather` divide
417
+ uncompressed NumPy payload bytes by elapsed time; they measure bytes accepted or
418
+ returned by the public API, not physical storage bandwidth. Write is page-cache
419
+ ingestion with durability deferred consistently across backends; call `sync()`
420
+ explicitly when measuring Zrecord durability. Disk size is allocated blocks,
421
+ not sparse apparent size.
422
+ Every backend name states its actual codec; the full default set is required
423
+ rather than silently skipped when a package is missing.
413
424
 
414
425
  **Machine** — one local workstation (AMD Ryzen AI 9 HX PRO 370, 12 cores / 24
415
426
  threads):
@@ -453,46 +464,67 @@ comparison; other record sizes are a `--shape` away.
453
464
 
454
465
  Realistic records — 256 KiB per record, 64 MiB per batch:
455
466
 
456
- | store | write | gather | on disk | ratio |
457
- |-------------------|------------|-------------|---------|-------|
458
- | zrecord-zstd | 911 MiB/s | 3987 MiB/s | 555 MiB | 1.13x |
459
- | zrecord-zstdict | 39 MiB/s | 1531 MiB/s | 388 MiB | 1.61x |
460
- | zrecord-raw | 2531 MiB/s | 7894 MiB/s | 625 MiB | 1.00x |
461
- | npy-mmap | 2403 MiB/s | 3418 MiB/s | 625 MiB | 1.00x |
462
- | hdf5 | 2517 MiB/s | 1871 MiB/s | 625 MiB | 1.00x |
463
- | blosc2 | 63 MiB/s | 543 MiB/s | 536 MiB | 1.17x |
464
- | tensorstore | 345 MiB/s | 268 MiB/s | 462 MiB | 1.35x |
467
+ Dense snapshot: `1.7.7`.
468
+
469
+ | store | logical write | logical gather | on disk | ratio |
470
+ |----------------------------|--------------:|---------------:|--------:|------:|
471
+ | zrecord-zstd | 1073 MiB/s | 4572 MiB/s | 555 MiB | 1.13x |
472
+ | zrecord-zstdict | 48 MiB/s | 2051 MiB/s | 388 MiB | 1.61x |
473
+ | zrecord-raw | 2325 MiB/s | 7996 MiB/s | 625 MiB | 1.00x |
474
+ | npy-mmap-raw | 2824 MiB/s | 3880 MiB/s | 625 MiB | 1.00x |
475
+ | hdf5-raw | 2385 MiB/s | 2155 MiB/s | 625 MiB | 1.00x |
476
+ | hdf5-gzip | 50 MiB/s | 234 MiB/s | 445 MiB | 1.40x |
477
+ | lmdb-raw | 794 MiB/s | 3406 MiB/s | 635 MiB | 0.98x |
478
+ | arrow-ipc-raw | 1657 MiB/s | 2601 MiB/s | 625 MiB | 1.00x |
479
+ | arrow-ipc-zstd | 311 MiB/s | 104 MiB/s | 418 MiB | 1.49x |
480
+ | parquet-raw | 1220 MiB/s | 442 MiB/s | 625 MiB | 1.00x |
481
+ | parquet-zstd | 311 MiB/s | 91 MiB/s | 418 MiB | 1.49x |
482
+ | arrayrecord-raw | 1878 MiB/s | 1873 MiB/s | 625 MiB | 1.00x |
483
+ | arrayrecord-zstd | 481 MiB/s | 1530 MiB/s | 567 MiB | 1.10x |
484
+ | tiledb-raw | 818 MiB/s | 780 MiB/s | 625 MiB | 1.00x |
485
+ | tiledb-zstd | 812 MiB/s | 811 MiB/s | 553 MiB | 1.13x |
465
486
 
466
487
  At 256 KiB per record, a 64 MiB batch exceeds L3 and the raw paths are primarily
467
- DRAM-bandwidth-bound. Zrecord-raw reaches 7.7 GiB/s and is 2.3x npy-mmap here;
468
- plain zstd gathers at 3.9 GiB/s. DenseStore demonstrates that typed record
469
- ownership and per-record compression do not turn fixed tensors into an object-
470
- store slow path.
471
-
472
- ### RaggedStore: Variable-Length Records
473
-
474
- This workload contains 100,000 int32 token sequences with lognormal lengths
475
- clipped to 16..2048 (observed median 253), totaling 142.7 MiB of logical payload.
476
- Each of 100 random batches contains 256 records. Every backend must return an
477
- ordered `list[np.ndarray]`; returning only flat values and offsets is not enough.
478
-
479
- | backend | write | gather | krecords/s | p95 | disk | ratio |
480
- |-------------------|------:|-------:|-----------:|----:|-----:|------:|
481
- | zrecord-zstd | 127 MiB/s | 317 MiB/s | 221.0 | 1.48 ms | 28.1 MiB | 5.08x |
482
- | zrecord-zstdict | 23 MiB/s | 314 MiB/s | 219.0 | 1.48 ms | 17.1 MiB | 8.32x |
483
- | zrecord-raw | 499 MiB/s | 313 MiB/s | 218.2 | 1.72 ms | 147.2 MiB | 0.97x |
484
- | packed-offsets | 1507 MiB/s | 976 MiB/s | 681.3 | 0.53 ms | 143.4 MiB | 0.99x |
485
- | hdf5-vlen | 36 MiB/s | 27 MiB/s | 18.9 | 20.01 ms | 146.4 MiB | 0.97x |
486
-
487
- `packed-offsets` is the useful raw lower bound: a specialized payload file plus
488
- one offsets array for one-dimensional tokens. It is not a feature-equivalent
489
- store: it has no compression, arbitrary-rank shape records, native typed owner,
490
- paired append publication, swap-delete, or compact. Zrecord pays for those
491
- semantics, while still gathering about 11.6x as many records/s as HDF5 vlen. The
492
- compressed paths gather at essentially raw Zrecord speed for this small-record
493
- workload; `zstdict` reduces 142.7 MiB to 17.1 MiB (8.32x) at 1.48 ms p95. Disk
488
+ DRAM-bandwidth-bound. Zrecord-raw reaches 7.8 GiB/s and is 2.1x npy-mmap-raw here;
489
+ plain zstd gathers at 4.5 GiB/s. LMDB and Arrow IPC are competitive raw record
490
+ stores, while codecs tied to whole IPC batches or Parquet row groups pay read
491
+ amplification on random gathers. DenseStore demonstrates that
492
+ typed record ownership and per-record compression do not turn fixed tensors
493
+ into an object-store slow path.
494
+
495
+ ### RaggedStore: Variable-Shape Records
496
+
497
+ This workload contains 2,500 variable-resolution CHW RGB images. Height and
498
+ width are independently lognormal and clipped to 64..512 (observed medians
499
+ 223 and 224), totaling 417.1 MiB of logical uint8 payload. Each of 50 random
500
+ batches contains 256 records. Every backend persists payload plus exact shape
501
+ and must return an ordered `list[np.ndarray]` of `(3, H, W)` arrays; a flat byte
502
+ list or a one-dimensional variable-length abstraction is not enough.
503
+
504
+ | backend | logical write | logical gather | krecords/s | p95 | disk | ratio |
505
+ |----------------------------|--------------:|---------------:|-----------:|----:|-----:|------:|
506
+ | zrecord-zstd | 2095 MiB/s | 5738 MiB/s | 34.6 | 8.17 ms | 27.1 MiB | 15.42x |
507
+ | zrecord-zstdict | 38 MiB/s | 6271 MiB/s | 37.8 | 7.64 ms | 17.8 MiB | 23.38x |
508
+ | zrecord-raw | 1493 MiB/s | 7017 MiB/s | 42.3 | 7.16 ms | 417.3 MiB | 1.00x |
509
+ | hdf5-raw | 1055 MiB/s | 1258 MiB/s | 7.6 | 40.55 ms | 418.0 MiB | 1.00x |
510
+ | hdf5-gzip | 243 MiB/s | 119 MiB/s | 0.7 | 406.67 ms | 30.0 MiB | 13.92x |
511
+ | lmdb-raw | 914 MiB/s | 6965 MiB/s | 42.0 | 6.63 ms | 422.2 MiB | 0.99x |
512
+ | arrow-ipc-raw | 1448 MiB/s | 3950 MiB/s | 23.8 | 13.03 ms | 417.2 MiB | 1.00x |
513
+ | arrow-ipc-zstd | 529 MiB/s | 181 MiB/s | 1.1 | 268.28 ms | 26.0 MiB | 16.02x |
514
+ | parquet-raw | 977 MiB/s | 466 MiB/s | 2.8 | 100.94 ms | 417.2 MiB | 1.00x |
515
+ | parquet-zstd | 510 MiB/s | 165 MiB/s | 1.0 | 288.08 ms | 26.0 MiB | 16.03x |
516
+ | arrayrecord-raw | 1417 MiB/s | 2612 MiB/s | 15.8 | 18.69 ms | 417.6 MiB | 1.00x |
517
+ | arrayrecord-zstd | 861 MiB/s | 1449 MiB/s | 8.7 | 39.32 ms | 27.3 MiB | 15.27x |
518
+ | tiledb-raw | 419 MiB/s | 50 MiB/s | 0.3 | 919.16 ms | 417.2 MiB | 1.00x |
519
+ | tiledb-zstd | 618 MiB/s | 124 MiB/s | 0.8 | 367.07 ms | 27.1 MiB | 15.40x |
520
+
521
+ LMDB is the fastest raw record contender and Arrow IPC is the strongest raw
522
+ typed-file alternative. Zrecord-zstd delivers 5.6 GiB/s of logical payload while
523
+ reducing the corpus to 27.1 MiB. HDF5, Arrow IPC, Parquet, ArrayRecord and TileDB
524
+ show the same framework/codec tradeoffs in both tables; compressed batch, chunk
525
+ and row-group formats pay read amplification on random records. Disk
494
526
  includes shape records, both loc entries, codec frames, dictionaries, and global
495
- headers. Arrow IPC was unavailable on this machine and was skipped.
527
+ headers where the format uses them.
496
528
 
497
529
  ### DataLoader: Fixed-Shape Pipeline
498
530
 
@@ -529,7 +561,9 @@ and `--transform augment`); they are not part of the default `all`:
529
561
  * **Sampler** — zsampler draws ~1.9–2x faster than free-threaded numpy (the
530
562
  256-row run was a noisy 2.46x), the same broad margin as on the GIL build: a
531
563
  batch is one Zig call either way.
532
- * **Store** — the free-threaded numbers track the GIL table closely.
564
+ * **Historical Store snapshot** — these older rows predate the symmetric
565
+ benchmark matrix and are retained only for the GIL/free-threaded comparison.
566
+ The free-threaded numbers track the GIL table closely.
533
567
  `zrecord-raw` gathers 8.9 vs 8.8 GiB/s and `zrecord-zstd` 4.7 vs 4.6 GiB/s.
534
568
  The no-GIL table was run without blosc2 so its import could not change the
535
569
  process mode. blosc2 was measured separately at 678 MiB/s; its extension
@@ -540,10 +574,10 @@ and `--transform augment`); they are not part of the default `all`:
540
574
  |--------------|--------------------|---------------------|
541
575
  | zrecord-raw | 8803 MiB/s | 8878 MiB/s |
542
576
  | zrecord-zstd | 4603 MiB/s | 4707 MiB/s |
543
- | npy-mmap | 4408 MiB/s | 3974 MiB/s |
544
- | hdf5 | 2288 MiB/s | 2210 MiB/s |
545
- | blosc2 | 646 MiB/s | 678 MiB/s (GIL enabled) |
546
- | tensorstore | 299 MiB/s | 323 MiB/s |
577
+ | npy-mmap-raw | 4408 MiB/s | 3974 MiB/s |
578
+ | hdf5-raw | 2288 MiB/s | 2210 MiB/s |
579
+ | blosc2-zstd | 646 MiB/s | 678 MiB/s (GIL enabled) |
580
+ | tensorstore-zarr3-gzip | 299 MiB/s | 323 MiB/s |
547
581
  * **Loader, identity** — unchanged on both interpreters at 64 MiB per batch:
548
582
 
549
583
  | loader | CPython 3.14 (GIL) | free-threaded 3.14t |
@@ -685,6 +719,7 @@ explicit memory bound while zrecord independently caps filesystem writes.
685
719
  zig build # host shared objects, into loaderx/lib/
686
720
  zig build test # native store suite, in both Debug and ReleaseFast
687
721
  python3 scripts/test_loaderx.py # Python integration suite against the real build
722
+ uv pip install -r scripts/requirements-bench.txt
688
723
  python3 scripts/bench_dense.py # fixed-shape store comparison
689
724
  python3 scripts/bench_ragged.py # variable-length store comparison
690
725
  python3 scripts/bench.py # machine, sampler, and dense loader layers