dataset-rt 0.3.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/.github/workflows/ci.yml +3 -0
  2. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/.gitignore +3 -0
  3. dataset_rt-0.4.0/.vscode/settings.json +3 -0
  4. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/AGENTS.md +8 -4
  5. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/CHANGELOG.md +105 -0
  6. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/Cargo.lock +37 -18
  7. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/Cargo.toml +5 -3
  8. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/PKG-INFO +24 -15
  9. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/README.md +10 -11
  10. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt/__init__.py +14 -0
  11. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt/_dataset_rt.pyi +15 -2
  12. dataset_rt-0.4.0/dataset_rt/api.py +88 -0
  13. dataset_rt-0.4.0/dataset_rt/benchmarks/__init__.py +1 -0
  14. dataset_rt-0.4.0/dataset_rt/benchmarks/config.py +79 -0
  15. dataset_rt-0.4.0/dataset_rt/benchmarks/distributed.py +679 -0
  16. dataset_rt-0.4.0/dataset_rt/benchmarks/resources.py +65 -0
  17. dataset_rt-0.4.0/dataset_rt/config.py +173 -0
  18. dataset_rt-0.4.0/dataset_rt/dataset.py +611 -0
  19. dataset_rt-0.4.0/dataset_rt/integrations/__init__.py +1 -0
  20. dataset_rt-0.4.0/dataset_rt/integrations/loader.py +485 -0
  21. dataset_rt-0.4.0/dataset_rt/integrations/loader_logging.py +30 -0
  22. dataset_rt-0.4.0/dataset_rt/integrations/loader_options.py +186 -0
  23. dataset_rt-0.4.0/dataset_rt/integrations/loading.py +247 -0
  24. dataset_rt-0.4.0/dataset_rt/integrations/torch.py +44 -0
  25. dataset_rt-0.4.0/dataset_rt/metadata.py +39 -0
  26. dataset_rt-0.4.0/dataset_rt/reconstruction.py +192 -0
  27. dataset_rt-0.4.0/dataset_rt/records.py +267 -0
  28. dataset_rt-0.4.0/dataset_rt/resources.py +204 -0
  29. dataset_rt-0.4.0/dataset_rt/run_benchmark.py +65 -0
  30. dataset_rt-0.4.0/dataset_rt/runtime.py +200 -0
  31. dataset_rt-0.4.0/dataset_rt/writer.py +503 -0
  32. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/architecture.md +15 -1
  33. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/determinism.md +4 -1
  34. dataset_rt-0.4.0/docs/fork-reproducer.md +151 -0
  35. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/inversion-analysis.md +15 -0
  36. dataset_rt-0.4.0/docs/inversion-checklists.md +643 -0
  37. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/python-api.md +271 -15
  38. dataset_rt-0.4.0/docs/remote-validation.md +172 -0
  39. dataset_rt-0.4.0/docs/runtime.md +560 -0
  40. dataset_rt-0.4.0/docs/storage-format.md +145 -0
  41. dataset_rt-0.4.0/examples/ddp_loading.py +137 -0
  42. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/justfile +4 -0
  43. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/pyproject.toml +20 -17
  44. dataset_rt-0.4.0/pyrefly.toml +31 -0
  45. dataset_rt-0.4.0/pytest.ini +6 -0
  46. dataset_rt-0.4.0/ruff.toml +33 -0
  47. dataset_rt-0.4.0/scripts/bench_parallel.py +188 -0
  48. dataset_rt-0.4.0/scripts/bench_reader.py +666 -0
  49. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/scripts/bench_smoke.py +4 -1
  50. dataset_rt-0.4.0/scripts/bench_writer.py +299 -0
  51. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/scripts/build_local.py +5 -2
  52. dataset_rt-0.4.0/scripts/check_precise_types.py +113 -0
  53. dataset_rt-0.4.0/scripts/repro_macos_fork.c +74 -0
  54. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/scripts/sync_api_docs.py +65 -26
  55. dataset_rt-0.4.0/src/dataset_rt/arrow_metadata/mapped.rs +120 -0
  56. dataset_rt-0.4.0/src/dataset_rt/arrow_metadata.rs +245 -0
  57. dataset_rt-0.4.0/src/dataset_rt/channel/tests.rs +222 -0
  58. dataset_rt-0.4.0/src/dataset_rt/channel.rs +184 -0
  59. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/dataset.rs +169 -20
  60. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/lib.rs +2 -0
  61. dataset_rt-0.4.0/src/dataset_rt/runtime/tests.rs +71 -0
  62. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/runtime.rs +23 -32
  63. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/samples_metadata.rs +157 -33
  64. dataset_rt-0.4.0/src/dataset_rt/storage/index.rs +117 -0
  65. dataset_rt-0.4.0/src/dataset_rt/storage/manifest.rs +155 -0
  66. dataset_rt-0.4.0/src/dataset_rt/storage/migration.rs +471 -0
  67. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/storage.rs +59 -63
  68. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/types.rs +5 -0
  69. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/worker_pool.rs +4 -4
  70. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer/pipeline.rs +7 -7
  71. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer.rs +9 -9
  72. dataset_rt-0.4.0/tests/fixtures/loader_logging_process.py +80 -0
  73. dataset_rt-0.4.0/tests/fixtures/manifest-v2/README.md +12 -0
  74. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/index.bin +0 -0
  75. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/manifest.json +67 -0
  76. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/metadata.arrow +0 -0
  77. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000000.bin +0 -0
  78. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000001.bin +0 -0
  79. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000002.bin +0 -0
  80. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000003.bin +0 -0
  81. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/index.bin +0 -0
  82. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/manifest.json +47 -0
  83. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/metadata.arrow +0 -0
  84. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/shards/000000.bin +0 -0
  85. dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/shards/000001.bin +0 -0
  86. dataset_rt-0.4.0/tests/fixtures/manifest-v2/provenance.json +146 -0
  87. dataset_rt-0.4.0/tests/fixtures/manifest-v2/saved-metadata.arrow +0 -0
  88. dataset_rt-0.4.0/tests/integrations/test_benchmark_module.py +251 -0
  89. dataset_rt-0.4.0/tests/integrations/test_loader_correctness.py +159 -0
  90. dataset_rt-0.4.0/tests/integrations/test_parallel_loader.py +912 -0
  91. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/integrations/test_pytorch.py +17 -6
  92. dataset_rt-0.4.0/tests/integrations/test_pytorch_distributed.py +560 -0
  93. dataset_rt-0.4.0/tests/integrations/test_reader_feasibility.py +311 -0
  94. dataset_rt-0.4.0/tests/integrations/test_serial_loader.py +681 -0
  95. dataset_rt-0.4.0/tests/test_benchmark_config.py +51 -0
  96. dataset_rt-0.4.0/tests/test_compatibility.py +168 -0
  97. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/test_dataset_rt.py +98 -43
  98. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/test_integrity.py +98 -17
  99. dataset_rt-0.4.0/tests/test_loader_logging.py +75 -0
  100. dataset_rt-0.4.0/tests/test_manifest_migration.py +680 -0
  101. dataset_rt-0.4.0/tests/test_parallel_writer.py +622 -0
  102. dataset_rt-0.4.0/tests/test_precise_types.py +43 -0
  103. dataset_rt-0.4.0/tests/test_reader_benchmark.py +162 -0
  104. dataset_rt-0.4.0/tests/test_reader_contracts.py +340 -0
  105. dataset_rt-0.4.0/tests/test_reconstruction.py +474 -0
  106. dataset_rt-0.4.0/tests/test_resources.py +232 -0
  107. dataset_rt-0.4.0/tests/test_writer_benchmark.py +74 -0
  108. dataset_rt-0.4.0/uv.lock +1427 -0
  109. dataset_rt-0.3.2/dataset_rt/api.py +0 -650
  110. dataset_rt-0.3.2/docs/runtime.md +0 -85
  111. dataset_rt-0.3.2/docs/storage-format.md +0 -92
  112. dataset_rt-0.3.2/uv.lock +0 -465
  113. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/.python-version +0 -0
  114. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/LICENSE +0 -0
  115. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt/py.typed +0 -0
  116. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt_spec.md +0 -0
  117. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/build.md +0 -0
  118. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/development.md +0 -0
  119. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/serialization.md +0 -0
  120. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/rust-toolchain.toml +0 -0
  121. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/compression.rs +0 -0
  122. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/dataset_runtime.rs +0 -0
  123. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/sampling.rs +0 -0
  124. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer/profiler.rs +0 -0
  125. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer/progress.rs +0 -0
  126. {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/test_api_docs.py +0 -0
@@ -44,6 +44,9 @@ jobs:
44
44
  - name: Ruff lint
45
45
  run: uv run --python 3.11 --extra dev ruff check dataset_rt tests scripts
46
46
 
47
+ - name: Precise type policy
48
+ run: uv run --python 3.11 --extra dev scripts/check_precise_types.py
49
+
47
50
  - name: Pyrefly
48
51
  run: uv run --python 3.11 --extra dev pyrefly check
49
52
 
@@ -13,7 +13,10 @@
13
13
  !py.typed
14
14
  !/LICENSE
15
15
  !/justfile
16
+ !/.vscode/settings.json
17
+ !/tests/fixtures/manifest-v2/**
16
18
 
17
19
  /.venv/
18
20
  /target/
19
21
  /dist/
22
+ /plan/
@@ -0,0 +1,3 @@
1
+ {
2
+ "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python"
3
+ }
@@ -14,7 +14,7 @@ Core vocabulary:
14
14
  - **Cache directory**: One immutable DatasetRT cache containing `manifest.json`, `metadata.arrow`, `index.bin`, and `shards/`.
15
15
  - **Manifest**: The publication marker for a complete cache. Readers reject caches without a valid manifest.
16
16
  - **Metadata**: Primitive sample-level values stored columnarly in `metadata.arrow` and redundantly embedded in each shard record.
17
- - **Physical sample**: A concrete `(cache_id, sample_id)` pair. `cache_id` is the position of the cache path passed to `CachedDataset`; `sample_id` is the row within that cache.
17
+ - **Physical sample**: A concrete `(cache_id, sample_id)` pair. V3 manifests persist `cache_id`; legacy v2 caches retain their supplied path positions until explicit migration. `sample_id` is the row within that cache. Rust maps cache IDs to internal array positions; never index a cache array directly by public ID.
18
18
  - **Samples metadata table**: A Polars table with `cache_id`, `sample_id`, metadata columns, and `weight`. Rust owns the authoritative weight vector and validates all updates.
19
19
  - **Epoch plan**: The ordered list of physical samples emitted by an iterator. With `shuffle=True`, it is deterministic weighted multinomial sampling with replacement for the dataset seed, epoch, and weight vector; it is not a permutation.
20
20
  - **Shard**: A binary file containing concatenated sample records. The index maps each sample to a shard id, offset, and byte length.
@@ -26,7 +26,7 @@ Important external tools:
26
26
  - **maturin**: Builds and installs the mixed Rust/Python package during development and release.
27
27
  - **Polars**: Columnar DataFrame engine used by the Python API for metadata-aware weight editing.
28
28
  - **Apache Arrow IPC**: Columnar file format used for `metadata.arrow` and in-memory samples metadata transfer.
29
- - **crossbeam-channel**: Bounded Rust channels used for reader/writer backpressure.
29
+ - **channel.rs**: Bounded Rust channels with operation-local mutex/condition-variable waits for reader/writer backpressure; fresh child runtimes must not use an inherited thread parker.
30
30
  - **indicatif**: Progress bar rendering for cache writes.
31
31
  - **lz4_flex**: Per-record LZ4 compression for random-access payload reads.
32
32
  - **uv**: Python environment and command runner. Use it for every Python command.
@@ -109,14 +109,18 @@ Use `uv run --python 3.11 --extra dev ...` for Python commands. Do not call bare
109
109
 
110
110
  Python package:
111
111
 
112
- - `dataset_rt/api.py` defines the public API, Pydantic configs, typed `NamedTuple` return values, and Polars weight-table helpers.
112
+ - `dataset_rt/api.py` and `__init__.py` are stable public facades; implementation modules never import them.
113
+ - `dataset_rt/config.py` defines frozen Pydantic configs; `records.py` defines payload records, source protocols, and typed outcomes. Public classes retain `dataset_rt.api` pickle paths.
114
+ - `dataset_rt/runtime.py` delegates native worker-pool ownership and cache writing; `dataset.py` delegates reading and authoritative dataset state.
115
+ - `dataset_rt/metadata.py` converts metadata through in-memory columnar Arrow IPC.
116
+ - `dataset_rt/integrations/torch.py` contains the lazy, optional legacy Torch adapter.
113
117
  - `dataset_rt/_dataset_rt.pyi` mirrors the Rust extension API for type checking.
114
118
  - `dataset_rt/__init__.py` re-exports the public surface.
115
119
 
116
120
  Rust modules:
117
121
 
118
122
  - `types.rs`: validated IDs, config newtypes, metadata enums, samples, and `CacheError`.
119
- - `storage.rs`: cache format, manifests, metadata Arrow files, indexes, shards, compression envelopes, checksums, and cache loading.
123
+ - `storage.rs`: cache format, metadata Arrow files, indexes, shards, compression envelopes, checksums, and cache loading. `storage/manifest.rs` contains separate v2/v3 records; `storage/migration.rs` upgrades manifests while preserving resolved IDs.
120
124
  - `writer.rs` and `writer/pipeline.rs`: source ingestion, bounded queues, serialization workers, ordered commit, progress, profiling, and publication.
121
125
  - `dataset.rs`: Python-facing `CachedDataset`, schema checks, weight state, weight validation, and iterator construction.
122
126
  - `runtime.rs`: Rust-owned synchronous reader pipeline with scheduler, workers, bounded result queue, and reorder buffer.
@@ -2,6 +2,111 @@
2
2
 
3
3
  All notable changes to DatasetRT are documented here.
4
4
 
5
+ ## 0.4.0 - 2026-10-15
6
+
7
+ This release consolidates the 0.4 release candidates since 0.3.2.
8
+
9
+ ### Highlights
10
+
11
+ - Add finite, metadata-aware PyTorch DataLoaders with persistent sampling streams,
12
+ automatic worker sizing, and distributed sample budgets.
13
+ - Add optional spawned cache-source writers and portable dataset reconstruction
14
+ settings that reference existing cache payloads.
15
+ - Reduce reader memory with shared immutable Arrow metadata and mapped indexes,
16
+ and bound outstanding reads together with queued and reordered results.
17
+
18
+ ### Backwards Incompatible Changes
19
+
20
+ - Write new caches with v3 manifests and persistent name-derived `cache_id`
21
+ values. Existing v2 caches remain readable with positional IDs. Treat IDs as
22
+ opaque; use `CachedDataset.update_manifests(version=3)` for explicit migration,
23
+ constructing legacy datasets in their original path order to preserve IDs.
24
+ Older releases that only read v2 cannot read migrated caches.
25
+ - Make `CachedDataset.to_torch_dataloader()` epochs finite instead of the
26
+ infinite streams in early release candidates. Set `samples_per_epoch` for a
27
+ global sample budget, or omit it to inherit `len(dataset)` at construction.
28
+ Iterate the loader directly instead of externally limiting an infinite stream.
29
+ `batch_size` must be a positive integer; use `1` for single-sample collation.
30
+ - Use explicit automatic modes for `num_workers`, `native_num_workers`,
31
+ `persistent_workers`, and `prefetch_factor` (`"auto"`), and for seeds
32
+ (`"random"`). Replace previous release-candidate `None` selections accordingly.
33
+ Sequential loading rejects an explicit seed. Automatic persistence is enabled
34
+ with loading workers; automatic prefetch uses two batches per worker and is
35
+ disabled with zero workers. Explicit persistence or integer prefetch with
36
+ zero workers raises an error.
37
+ - Default loading processes to `multiprocessing_context="spawn"`. Use a guarded
38
+ application entry point and picklable callbacks. Replace release-candidate
39
+ `None` contexts with `"inherit"` to select Python's platform default.
40
+ - Default shuffled loading to replicated populations for global weighted
41
+ sampling; sequential loading defaults to split populations. Explicit
42
+ `worker_partition="split"` remains available, but equal consumer quotas can
43
+ approximate the global weight distribution. Replication retains private
44
+ weights and sampling state per consumer; 10M f64 weights require 80 MB per
45
+ consumer, in addition to sampling state.
46
+ - Divide global sample budgets equally across initialized DDP ranks, padding
47
+ with `drop_last=False` or discarding with `drop_last=True` to complete local
48
+ batches. Rank zero warns when the effective budget changes. Rank lengths and
49
+ reported loader lengths match emitted batches.
50
+ - Replace release-candidate v1 reconstruction bundles containing
51
+ `metadata.parquet` with v2 bundles containing `metadata.arrow`. Recreate old
52
+ exports with `CachedDataset.dump_config()` before calling `from_config()`.
53
+
54
+ ### New Features
55
+
56
+ - Add `WriterConfig.num_processes` for bounded spawned source writers with
57
+ input-order outcomes; zero retains serial writing. Sources must be picklable
58
+ and importable. Worker death or timeout stops processing without retries.
59
+ - Add `CachedDataset.dump_config()` and `CachedDataset.from_config()` for
60
+ reconstruction settings and immutable active metadata snapshots. Loader
61
+ `work_dir` controls retained snapshots; the default manages temporary output.
62
+ - Add `python -m dataset_rt.run_benchmark` with the `benchmark` extra for local
63
+ rank launch or external `torchrun`, generated or existing caches, CPU/Gloo or
64
+ CUDA/NCCL execution, correctness checks, and JSON resource/performance reports.
65
+
66
+ ### Improvements
67
+
68
+ - Resolve automatic loader counts from CPU affinity and visible container quota
69
+ and CPU-set limits, accounting for local ranks. Choose up to four loading
70
+ processes and 1–4 native threads per consumer; a one-CPU budget uses zero
71
+ loading processes. Set explicit counts for a reproducible worker topology.
72
+ - Log loader configuration and reader lifecycle through `dataset_rt.loader`.
73
+ `verbose=False` suppresses diagnostics while retaining the warning that
74
+ explicitly nonpersistent workers repeat their sampling stream each epoch.
75
+ - Validate worker options, finite timeouts, sample budgets, and multiprocess
76
+ transform picklability before exporting loader reconstruction artifacts.
77
+
78
+ ### Bug Fixes
79
+
80
+ - Continue sampling across short epochs by retaining loading workers by default.
81
+ Explicit `persistent_workers=False` still restarts reader progress and now
82
+ warns about repeated samples.
83
+ - Preserve bounded reader memory when an early read completes late, while
84
+ retaining planned emission order and iterator-drop cancellation.
85
+ - Allow fresh native runtimes after macOS `fork` without reusing an inherited
86
+ thread parker; bounded channels retain FIFO delivery and backpressure.
87
+ - Assign whole-batch worker quotas so `len(loader)` matches emitted batches and
88
+ `drop_last=True` drops at most one local tail.
89
+
90
+ ### Performance
91
+
92
+ - Share immutable memory-mapped Arrow metadata, selected row views, cache
93
+ indexes, and shuffled split permutations across consumers on one host.
94
+ Weights and sampling state remain private. Avoid whole-table IPC copies in
95
+ metadata access and updates, and stream reconstruction exports in bounded
96
+ chunks.
97
+
98
+ ### Developers
99
+
100
+ - Add paired reader/writer benchmarks, CPU DDP acceptance examples, and
101
+ `just test-correctness` for fast behavioral checks with spawned workers.
102
+ - Preserve public imports and pickle paths while splitting Python modules;
103
+ enforce precise annotations and run ordinary tests with pytest-xdist.
104
+
105
+ ### Validation coverage
106
+
107
+ - CPU acceptance is verified. CUDA/NCCL and physical multi-host acceptance
108
+ remain planned validation work on suitable hardware.
109
+
5
110
  ## 0.3.2 - 2026-09-28
6
111
 
7
112
  ### Added
@@ -82,6 +82,23 @@ name = "arrow-schema"
82
82
  version = "54.3.1"
83
83
  source = "registry+https://github.com/rust-lang/crates.io-index"
84
84
  checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9"
85
+ dependencies = [
86
+ "bitflags 2.13.2",
87
+ ]
88
+
89
+ [[package]]
90
+ name = "arrow-select"
91
+ version = "54.3.1"
92
+ source = "registry+https://github.com/rust-lang/crates.io-index"
93
+ checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619"
94
+ dependencies = [
95
+ "ahash",
96
+ "arrow-array",
97
+ "arrow-buffer",
98
+ "arrow-data",
99
+ "arrow-schema",
100
+ "num",
101
+ ]
85
102
 
86
103
  [[package]]
87
104
  name = "autocfg"
@@ -95,6 +112,12 @@ version = "1.3.2"
95
112
  source = "registry+https://github.com/rust-lang/crates.io-index"
96
113
  checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
97
114
 
115
+ [[package]]
116
+ name = "bitflags"
117
+ version = "2.13.2"
118
+ source = "registry+https://github.com/rust-lang/crates.io-index"
119
+ checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
120
+
98
121
  [[package]]
99
122
  name = "block-buffer"
100
123
  version = "0.10.4"
@@ -190,21 +213,6 @@ dependencies = [
190
213
  "libc",
191
214
  ]
192
215
 
193
- [[package]]
194
- name = "crossbeam-channel"
195
- version = "0.5.16"
196
- source = "registry+https://github.com/rust-lang/crates.io-index"
197
- checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
198
- dependencies = [
199
- "crossbeam-utils",
200
- ]
201
-
202
- [[package]]
203
- name = "crossbeam-utils"
204
- version = "0.8.22"
205
- source = "registry+https://github.com/rust-lang/crates.io-index"
206
- checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
207
-
208
216
  [[package]]
209
217
  name = "crunchy"
210
218
  version = "0.2.4"
@@ -223,15 +231,17 @@ dependencies = [
223
231
 
224
232
  [[package]]
225
233
  name = "dataset_rt"
226
- version = "0.3.2"
234
+ version = "0.4.0"
227
235
  dependencies = [
228
236
  "arrow-array",
237
+ "arrow-buffer",
229
238
  "arrow-ipc",
230
239
  "arrow-schema",
231
- "crossbeam-channel",
240
+ "arrow-select",
232
241
  "hex",
233
242
  "indicatif",
234
243
  "lz4_flex",
244
+ "memmap2",
235
245
  "pyo3",
236
246
  "rand",
237
247
  "rand_chacha",
@@ -269,7 +279,7 @@ version = "24.12.23"
269
279
  source = "registry+https://github.com/rust-lang/crates.io-index"
270
280
  checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096"
271
281
  dependencies = [
272
- "bitflags",
282
+ "bitflags 1.3.2",
273
283
  "rustc_version",
274
284
  ]
275
285
 
@@ -447,6 +457,15 @@ version = "2.8.3"
447
457
  source = "registry+https://github.com/rust-lang/crates.io-index"
448
458
  checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
449
459
 
460
+ [[package]]
461
+ name = "memmap2"
462
+ version = "0.9.11"
463
+ source = "registry+https://github.com/rust-lang/crates.io-index"
464
+ checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
465
+ dependencies = [
466
+ "libc",
467
+ ]
468
+
450
469
  [[package]]
451
470
  name = "num"
452
471
  version = "0.4.3"
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "dataset_rt"
3
- version = "0.3.2"
3
+ version = "0.4.0"
4
4
  edition = "2021"
5
5
  description = "Framework-independent dataset cache runtime"
6
6
  license = "MIT"
@@ -15,13 +15,15 @@ path = "src/dataset_rt/lib.rs"
15
15
  crate-type = ["cdylib"]
16
16
 
17
17
  [dependencies]
18
- arrow-array = "54"
18
+ arrow-array = { version = "54", features = ["ffi"] }
19
19
  arrow-ipc = "54"
20
+ arrow-buffer = "54"
20
21
  arrow-schema = "54"
21
- crossbeam-channel = "0.5"
22
+ arrow-select = "54"
22
23
  hex = "0.4"
23
24
  indicatif = "0.18"
24
25
  lz4_flex = "0.14"
26
+ memmap2 = "0.9"
25
27
  pyo3 = { version = "0.29", features = ["abi3-py310", "extension-module"] }
26
28
  rand = "0.8"
27
29
  rand_chacha = "0.3"
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataset-rt
3
- Version: 0.3.2
4
- Classifier: Development Status :: 3 - Alpha
3
+ Version: 0.4.0
4
+ Classifier: Development Status :: 5 - Production/Stable
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Intended Audience :: Science/Research
7
7
  Classifier: License :: OSI Approved :: MIT License
@@ -15,11 +15,21 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
15
15
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
16
  Requires-Dist: pydantic>=2
17
17
  Requires-Dist: polars>=1
18
+ Requires-Dist: pyarrow>=19
19
+ Requires-Dist: numpy>=1.26
20
+ Requires-Dist: pydantic-settings>=2 ; extra == 'benchmark'
21
+ Requires-Dist: torch>=2.14.1 ; extra == 'benchmark'
22
+ Requires-Dist: psutil>=7.2.2 ; extra == 'benchmark'
18
23
  Requires-Dist: maturin[zig]>=1.7,<2 ; extra == 'dev'
19
24
  Requires-Dist: pytest>=8 ; extra == 'dev'
20
- Requires-Dist: ruff>=0.5 ; extra == 'dev'
21
- Requires-Dist: pyrefly>=0.21 ; extra == 'dev'
25
+ Requires-Dist: pytest-xdist>=3.8 ; extra == 'dev'
26
+ Requires-Dist: ruff>=0.16.9 ; extra == 'dev'
27
+ Requires-Dist: pyrefly>=1.3.1 ; extra == 'dev'
22
28
  Requires-Dist: pydantic-settings>=2 ; extra == 'dev'
29
+ Requires-Dist: torch>=2.14.1 ; extra == 'dev'
30
+ Requires-Dist: psutil>=7.2.2 ; extra == 'dev'
31
+ Requires-Dist: types-psutil>=7.2.2 ; extra == 'dev'
32
+ Provides-Extra: benchmark
23
33
  Provides-Extra: dev
24
34
  License-File: LICENSE
25
35
  Summary: Framework-independent dataset cache runtime
@@ -108,7 +118,7 @@ dataset.update_metadata(balanced)
108
118
  `index.bin`, shards, or manifests. Future iterators use the updated active table;
109
119
  already-created iterators keep their snapshot.
110
120
 
111
- With `ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
121
+ With `drt.ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
112
122
  multinomial sampling with replacement over the active table. With
113
123
  `shuffle=False`, it emits active rows in cyclic table order, including
114
124
  duplicates.
@@ -120,6 +130,10 @@ length to the new table row count. Call `set_epoch_len` afterward to override it
120
130
 
121
131
  ## Installation
122
132
 
133
+ For GPU and distributed loader measurements, see the
134
+ [portable benchmark commands](docs/remote-validation.md):
135
+ `python -m dataset_rt.run_benchmark --ranks 4 --num-workers 8 --device cuda`.
136
+
123
137
  DatasetRT supports Python 3.10 through 3.13.
124
138
 
125
139
  ```bash
@@ -140,12 +154,7 @@ primitive metadata.
140
154
  ```python
141
155
  from pathlib import Path
142
156
 
143
- from dataset_rt import (
144
- CacheInput,
145
- CacheSourcesDatasetSuccess,
146
- DatasetRuntime,
147
- ReaderConfig,
148
- )
157
+ import dataset_rt as drt
149
158
 
150
159
 
151
160
  class Images:
@@ -153,21 +162,21 @@ class Images:
153
162
 
154
163
  def __iter__(self):
155
164
  for image_id, image_bytes, label in load_my_images():
156
- yield CacheInput(
165
+ yield drt.CacheInput(
157
166
  data=image_bytes,
158
167
  metadata={"image_id": image_id, "label": label},
159
168
  )
160
169
 
161
170
 
162
- runtime = DatasetRuntime(num_workers=4)
171
+ runtime = drt.DatasetRuntime(num_workers=4)
163
172
  result = runtime.from_cache_sources(
164
173
  Images(),
165
174
  Path("cache"),
166
- reader_config=ReaderConfig(seed=42, shuffle=True),
175
+ reader_config=drt.ReaderConfig(seed=42, shuffle=True),
167
176
  )
168
177
 
169
178
  match result:
170
- case CacheSourcesDatasetSuccess(dataset, results):
179
+ case drt.CacheSourcesDatasetSuccess(dataset, results):
171
180
  pass
172
181
  case error:
173
182
  raise RuntimeError(error)
@@ -77,7 +77,7 @@ dataset.update_metadata(balanced)
77
77
  `index.bin`, shards, or manifests. Future iterators use the updated active table;
78
78
  already-created iterators keep their snapshot.
79
79
 
80
- With `ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
80
+ With `drt.ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
81
81
  multinomial sampling with replacement over the active table. With
82
82
  `shuffle=False`, it emits active rows in cyclic table order, including
83
83
  duplicates.
@@ -89,6 +89,10 @@ length to the new table row count. Call `set_epoch_len` afterward to override it
89
89
 
90
90
  ## Installation
91
91
 
92
+ For GPU and distributed loader measurements, see the
93
+ [portable benchmark commands](docs/remote-validation.md):
94
+ `python -m dataset_rt.run_benchmark --ranks 4 --num-workers 8 --device cuda`.
95
+
92
96
  DatasetRT supports Python 3.10 through 3.13.
93
97
 
94
98
  ```bash
@@ -109,12 +113,7 @@ primitive metadata.
109
113
  ```python
110
114
  from pathlib import Path
111
115
 
112
- from dataset_rt import (
113
- CacheInput,
114
- CacheSourcesDatasetSuccess,
115
- DatasetRuntime,
116
- ReaderConfig,
117
- )
116
+ import dataset_rt as drt
118
117
 
119
118
 
120
119
  class Images:
@@ -122,21 +121,21 @@ class Images:
122
121
 
123
122
  def __iter__(self):
124
123
  for image_id, image_bytes, label in load_my_images():
125
- yield CacheInput(
124
+ yield drt.CacheInput(
126
125
  data=image_bytes,
127
126
  metadata={"image_id": image_id, "label": label},
128
127
  )
129
128
 
130
129
 
131
- runtime = DatasetRuntime(num_workers=4)
130
+ runtime = drt.DatasetRuntime(num_workers=4)
132
131
  result = runtime.from_cache_sources(
133
132
  Images(),
134
133
  Path("cache"),
135
- reader_config=ReaderConfig(seed=42, shuffle=True),
134
+ reader_config=drt.ReaderConfig(seed=42, shuffle=True),
136
135
  )
137
136
 
138
137
  match result:
139
- case CacheSourcesDatasetSuccess(dataset, results):
138
+ case drt.CacheSourcesDatasetSuccess(dataset, results):
140
139
  pass
141
140
  case error:
142
141
  raise RuntimeError(error)
@@ -1,4 +1,5 @@
1
1
  from dataset_rt.api import (
2
+ AbsentManifestTarget,
2
3
  CachedDataset,
3
4
  CachedSample,
4
5
  CacheInput,
@@ -10,7 +11,13 @@ from dataset_rt.api import (
10
11
  CacheWriteResult,
11
12
  CacheWriteSuccess,
12
13
  DatasetRuntime,
14
+ Err,
15
+ ManifestUpdateEntry,
16
+ ManifestUpdateError,
17
+ ManifestUpdateReport,
18
+ Ok,
13
19
  ReaderConfig,
20
+ Result,
14
21
  ShardCompression,
15
22
  SizedTorchIterableDataset,
16
23
  WriterConfig,
@@ -34,4 +41,11 @@ __all__ = [
34
41
  "SizedTorchIterableDataset",
35
42
  "WriterConfig",
36
43
  "WriterProfilerConfig",
44
+ "AbsentManifestTarget",
45
+ "Err",
46
+ "ManifestUpdateEntry",
47
+ "ManifestUpdateError",
48
+ "ManifestUpdateReport",
49
+ "Ok",
50
+ "Result",
37
51
  ]
@@ -1,6 +1,11 @@
1
1
  from collections.abc import Iterator, Sequence
2
2
  from typing import NamedTuple
3
3
 
4
+ from typing_extensions import CapsuleType
5
+
6
+ from dataset_rt.config import WriterConfig
7
+ from dataset_rt.records import CacheSource
8
+
4
9
  MetadataValue = bool | int | float | str
5
10
 
6
11
  class CacheWriteRecord(NamedTuple):
@@ -27,15 +32,23 @@ class CachedDataset:
27
32
  self, cache_id: int, sample_id: int
28
33
  ) -> tuple[bytes, dict[str, MetadataValue], int, int]: ...
29
34
  def set_epoch_len(self, epoch_len: int) -> None: ...
35
+ def update_manifests(
36
+ self, version: int
37
+ ) -> tuple[bool, list[tuple[str, int, str]], str | None, str]: ...
30
38
  def metadata_ipc(self) -> bytes: ...
39
+ def metadata_arrow(self) -> CapsuleType: ...
40
+ def export_metadata_arrow(self, path: str) -> int: ...
41
+ def update_metadata_arrow(
42
+ self, metadata: CapsuleType, selection: CapsuleType | None = None
43
+ ) -> None: ...
31
44
  def update_metadata_ipc(self, ipc: bytes) -> None: ...
32
45
  def samples_metadata_ipc(self) -> bytes: ...
33
46
  def set_samples_metadata_ipc(self, ipc: bytes) -> None: ...
34
47
 
35
48
  def write_cache(
36
49
  runtime: DatasetRuntime,
37
- sources: object,
50
+ sources: CacheSource | list[CacheSource],
38
51
  base_cache_dir: str,
39
- writer_config: object,
52
+ writer_config: WriterConfig,
40
53
  reuse_existing: bool,
41
54
  ) -> list[CacheWriteRecord]: ...
@@ -0,0 +1,88 @@
1
+ """Stable public facade retaining historical imports and pickle lookup paths."""
2
+
3
+ from dataset_rt.config import (
4
+ DEFAULT_SHARD_COMPRESSION as DEFAULT_SHARD_COMPRESSION,
5
+ )
6
+ from dataset_rt.config import (
7
+ DEFAULT_WRITER_CONFIG as DEFAULT_WRITER_CONFIG,
8
+ )
9
+ from dataset_rt.config import (
10
+ DEFAULT_WRITER_PROFILER_CONFIG as DEFAULT_WRITER_PROFILER_CONFIG,
11
+ )
12
+ from dataset_rt.config import (
13
+ CompressionAlgo as CompressionAlgo,
14
+ )
15
+ from dataset_rt.config import (
16
+ ReaderConfig as ReaderConfig,
17
+ )
18
+ from dataset_rt.config import (
19
+ ShardCompression as ShardCompression,
20
+ )
21
+ from dataset_rt.config import (
22
+ WriterConfig as WriterConfig,
23
+ )
24
+ from dataset_rt.config import (
25
+ WriterProfilerConfig as WriterProfilerConfig,
26
+ )
27
+ from dataset_rt.dataset import CachedDataset as CachedDataset
28
+ from dataset_rt.records import (
29
+ AbsentManifestTarget as AbsentManifestTarget,
30
+ )
31
+ from dataset_rt.records import (
32
+ BytesLike as BytesLike,
33
+ )
34
+ from dataset_rt.records import (
35
+ CachedSample as CachedSample,
36
+ )
37
+ from dataset_rt.records import (
38
+ CacheInput as CacheInput,
39
+ )
40
+ from dataset_rt.records import (
41
+ CacheSource as CacheSource,
42
+ )
43
+ from dataset_rt.records import (
44
+ CacheSourcesDatasetError as CacheSourcesDatasetError,
45
+ )
46
+ from dataset_rt.records import (
47
+ CacheSourcesDatasetResult as CacheSourcesDatasetResult,
48
+ )
49
+ from dataset_rt.records import (
50
+ CacheSourcesDatasetSuccess as CacheSourcesDatasetSuccess,
51
+ )
52
+ from dataset_rt.records import (
53
+ CacheWriteError as CacheWriteError,
54
+ )
55
+ from dataset_rt.records import (
56
+ CacheWriteResult as CacheWriteResult,
57
+ )
58
+ from dataset_rt.records import (
59
+ CacheWriteSuccess as CacheWriteSuccess,
60
+ )
61
+ from dataset_rt.records import (
62
+ Err as Err,
63
+ )
64
+ from dataset_rt.records import (
65
+ ManifestUpdateEntry as ManifestUpdateEntry,
66
+ )
67
+ from dataset_rt.records import (
68
+ ManifestUpdateError as ManifestUpdateError,
69
+ )
70
+ from dataset_rt.records import (
71
+ ManifestUpdateReport as ManifestUpdateReport,
72
+ )
73
+ from dataset_rt.records import (
74
+ Metadata as Metadata,
75
+ )
76
+ from dataset_rt.records import (
77
+ MetadataValue as MetadataValue,
78
+ )
79
+ from dataset_rt.records import (
80
+ Ok as Ok,
81
+ )
82
+ from dataset_rt.records import (
83
+ Result as Result,
84
+ )
85
+ from dataset_rt.records import (
86
+ SizedTorchIterableDataset as SizedTorchIterableDataset,
87
+ )
88
+ from dataset_rt.runtime import DatasetRuntime as DatasetRuntime
@@ -0,0 +1 @@
1
+ """Optional benchmark machinery; importing DatasetRT does not import its dependencies."""