dataset-rt 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/.github/workflows/ci.yml +3 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/.gitignore +3 -0
- dataset_rt-0.4.0/.vscode/settings.json +3 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/AGENTS.md +8 -4
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/CHANGELOG.md +105 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/Cargo.lock +37 -18
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/Cargo.toml +5 -3
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/PKG-INFO +24 -15
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/README.md +10 -11
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt/__init__.py +14 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt/_dataset_rt.pyi +15 -2
- dataset_rt-0.4.0/dataset_rt/api.py +88 -0
- dataset_rt-0.4.0/dataset_rt/benchmarks/__init__.py +1 -0
- dataset_rt-0.4.0/dataset_rt/benchmarks/config.py +79 -0
- dataset_rt-0.4.0/dataset_rt/benchmarks/distributed.py +679 -0
- dataset_rt-0.4.0/dataset_rt/benchmarks/resources.py +65 -0
- dataset_rt-0.4.0/dataset_rt/config.py +173 -0
- dataset_rt-0.4.0/dataset_rt/dataset.py +611 -0
- dataset_rt-0.4.0/dataset_rt/integrations/__init__.py +1 -0
- dataset_rt-0.4.0/dataset_rt/integrations/loader.py +485 -0
- dataset_rt-0.4.0/dataset_rt/integrations/loader_logging.py +30 -0
- dataset_rt-0.4.0/dataset_rt/integrations/loader_options.py +186 -0
- dataset_rt-0.4.0/dataset_rt/integrations/loading.py +247 -0
- dataset_rt-0.4.0/dataset_rt/integrations/torch.py +44 -0
- dataset_rt-0.4.0/dataset_rt/metadata.py +39 -0
- dataset_rt-0.4.0/dataset_rt/reconstruction.py +192 -0
- dataset_rt-0.4.0/dataset_rt/records.py +267 -0
- dataset_rt-0.4.0/dataset_rt/resources.py +204 -0
- dataset_rt-0.4.0/dataset_rt/run_benchmark.py +65 -0
- dataset_rt-0.4.0/dataset_rt/runtime.py +200 -0
- dataset_rt-0.4.0/dataset_rt/writer.py +503 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/architecture.md +15 -1
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/determinism.md +4 -1
- dataset_rt-0.4.0/docs/fork-reproducer.md +151 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/inversion-analysis.md +15 -0
- dataset_rt-0.4.0/docs/inversion-checklists.md +643 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/python-api.md +271 -15
- dataset_rt-0.4.0/docs/remote-validation.md +172 -0
- dataset_rt-0.4.0/docs/runtime.md +560 -0
- dataset_rt-0.4.0/docs/storage-format.md +145 -0
- dataset_rt-0.4.0/examples/ddp_loading.py +137 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/justfile +4 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/pyproject.toml +20 -17
- dataset_rt-0.4.0/pyrefly.toml +31 -0
- dataset_rt-0.4.0/pytest.ini +6 -0
- dataset_rt-0.4.0/ruff.toml +33 -0
- dataset_rt-0.4.0/scripts/bench_parallel.py +188 -0
- dataset_rt-0.4.0/scripts/bench_reader.py +666 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/scripts/bench_smoke.py +4 -1
- dataset_rt-0.4.0/scripts/bench_writer.py +299 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/scripts/build_local.py +5 -2
- dataset_rt-0.4.0/scripts/check_precise_types.py +113 -0
- dataset_rt-0.4.0/scripts/repro_macos_fork.c +74 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/scripts/sync_api_docs.py +65 -26
- dataset_rt-0.4.0/src/dataset_rt/arrow_metadata/mapped.rs +120 -0
- dataset_rt-0.4.0/src/dataset_rt/arrow_metadata.rs +245 -0
- dataset_rt-0.4.0/src/dataset_rt/channel/tests.rs +222 -0
- dataset_rt-0.4.0/src/dataset_rt/channel.rs +184 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/dataset.rs +169 -20
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/lib.rs +2 -0
- dataset_rt-0.4.0/src/dataset_rt/runtime/tests.rs +71 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/runtime.rs +23 -32
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/samples_metadata.rs +157 -33
- dataset_rt-0.4.0/src/dataset_rt/storage/index.rs +117 -0
- dataset_rt-0.4.0/src/dataset_rt/storage/manifest.rs +155 -0
- dataset_rt-0.4.0/src/dataset_rt/storage/migration.rs +471 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/storage.rs +59 -63
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/types.rs +5 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/worker_pool.rs +4 -4
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer/pipeline.rs +7 -7
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer.rs +9 -9
- dataset_rt-0.4.0/tests/fixtures/loader_logging_process.py +80 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/README.md +12 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/index.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/manifest.json +67 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/metadata.arrow +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000000.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000001.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000002.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-plain/shards/000003.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/index.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/manifest.json +47 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/metadata.arrow +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/shards/000000.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/legacy-/303/251/shards/000001.bin +0 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/provenance.json +146 -0
- dataset_rt-0.4.0/tests/fixtures/manifest-v2/saved-metadata.arrow +0 -0
- dataset_rt-0.4.0/tests/integrations/test_benchmark_module.py +251 -0
- dataset_rt-0.4.0/tests/integrations/test_loader_correctness.py +159 -0
- dataset_rt-0.4.0/tests/integrations/test_parallel_loader.py +912 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/integrations/test_pytorch.py +17 -6
- dataset_rt-0.4.0/tests/integrations/test_pytorch_distributed.py +560 -0
- dataset_rt-0.4.0/tests/integrations/test_reader_feasibility.py +311 -0
- dataset_rt-0.4.0/tests/integrations/test_serial_loader.py +681 -0
- dataset_rt-0.4.0/tests/test_benchmark_config.py +51 -0
- dataset_rt-0.4.0/tests/test_compatibility.py +168 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/test_dataset_rt.py +98 -43
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/test_integrity.py +98 -17
- dataset_rt-0.4.0/tests/test_loader_logging.py +75 -0
- dataset_rt-0.4.0/tests/test_manifest_migration.py +680 -0
- dataset_rt-0.4.0/tests/test_parallel_writer.py +622 -0
- dataset_rt-0.4.0/tests/test_precise_types.py +43 -0
- dataset_rt-0.4.0/tests/test_reader_benchmark.py +162 -0
- dataset_rt-0.4.0/tests/test_reader_contracts.py +340 -0
- dataset_rt-0.4.0/tests/test_reconstruction.py +474 -0
- dataset_rt-0.4.0/tests/test_resources.py +232 -0
- dataset_rt-0.4.0/tests/test_writer_benchmark.py +74 -0
- dataset_rt-0.4.0/uv.lock +1427 -0
- dataset_rt-0.3.2/dataset_rt/api.py +0 -650
- dataset_rt-0.3.2/docs/runtime.md +0 -85
- dataset_rt-0.3.2/docs/storage-format.md +0 -92
- dataset_rt-0.3.2/uv.lock +0 -465
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/.python-version +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/LICENSE +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt/py.typed +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/dataset_rt_spec.md +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/build.md +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/development.md +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/docs/serialization.md +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/rust-toolchain.toml +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/compression.rs +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/dataset_runtime.rs +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/sampling.rs +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer/profiler.rs +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/src/dataset_rt/writer/progress.rs +0 -0
- {dataset_rt-0.3.2 → dataset_rt-0.4.0}/tests/test_api_docs.py +0 -0
|
@@ -44,6 +44,9 @@ jobs:
|
|
|
44
44
|
- name: Ruff lint
|
|
45
45
|
run: uv run --python 3.11 --extra dev ruff check dataset_rt tests scripts
|
|
46
46
|
|
|
47
|
+
- name: Precise type policy
|
|
48
|
+
run: uv run --python 3.11 --extra dev scripts/check_precise_types.py
|
|
49
|
+
|
|
47
50
|
- name: Pyrefly
|
|
48
51
|
run: uv run --python 3.11 --extra dev pyrefly check
|
|
49
52
|
|
|
@@ -14,7 +14,7 @@ Core vocabulary:
|
|
|
14
14
|
- **Cache directory**: One immutable DatasetRT cache containing `manifest.json`, `metadata.arrow`, `index.bin`, and `shards/`.
|
|
15
15
|
- **Manifest**: The publication marker for a complete cache. Readers reject caches without a valid manifest.
|
|
16
16
|
- **Metadata**: Primitive sample-level values stored columnarly in `metadata.arrow` and redundantly embedded in each shard record.
|
|
17
|
-
- **Physical sample**: A concrete `(cache_id, sample_id)` pair. `cache_id
|
|
17
|
+
- **Physical sample**: A concrete `(cache_id, sample_id)` pair. V3 manifests persist `cache_id`; legacy v2 caches retain their supplied path positions until explicit migration. `sample_id` is the row within that cache. Rust maps cache IDs to internal array positions; never index a cache array directly by public ID.
|
|
18
18
|
- **Samples metadata table**: A Polars table with `cache_id`, `sample_id`, metadata columns, and `weight`. Rust owns the authoritative weight vector and validates all updates.
|
|
19
19
|
- **Epoch plan**: The ordered list of physical samples emitted by an iterator. With `shuffle=True`, it is deterministic weighted multinomial sampling with replacement for the dataset seed, epoch, and weight vector; it is not a permutation.
|
|
20
20
|
- **Shard**: A binary file containing concatenated sample records. The index maps each sample to a shard id, offset, and byte length.
|
|
@@ -26,7 +26,7 @@ Important external tools:
|
|
|
26
26
|
- **maturin**: Builds and installs the mixed Rust/Python package during development and release.
|
|
27
27
|
- **Polars**: Columnar DataFrame engine used by the Python API for metadata-aware weight editing.
|
|
28
28
|
- **Apache Arrow IPC**: Columnar file format used for `metadata.arrow` and in-memory samples metadata transfer.
|
|
29
|
-
- **
|
|
29
|
+
- **channel.rs**: Bounded Rust channels with operation-local mutex/condition-variable waits for reader/writer backpressure; fresh child runtimes must not use an inherited thread parker.
|
|
30
30
|
- **indicatif**: Progress bar rendering for cache writes.
|
|
31
31
|
- **lz4_flex**: Per-record LZ4 compression for random-access payload reads.
|
|
32
32
|
- **uv**: Python environment and command runner. Use it for every Python command.
|
|
@@ -109,14 +109,18 @@ Use `uv run --python 3.11 --extra dev ...` for Python commands. Do not call bare
|
|
|
109
109
|
|
|
110
110
|
Python package:
|
|
111
111
|
|
|
112
|
-
- `dataset_rt/api.py`
|
|
112
|
+
- `dataset_rt/api.py` and `__init__.py` are stable public facades; implementation modules never import them.
|
|
113
|
+
- `dataset_rt/config.py` defines frozen Pydantic configs; `records.py` defines payload records, source protocols, and typed outcomes. Public classes retain `dataset_rt.api` pickle paths.
|
|
114
|
+
- `dataset_rt/runtime.py` delegates native worker-pool ownership and cache writing; `dataset.py` delegates reading and authoritative dataset state.
|
|
115
|
+
- `dataset_rt/metadata.py` converts metadata through in-memory columnar Arrow IPC.
|
|
116
|
+
- `dataset_rt/integrations/torch.py` contains the lazy, optional legacy Torch adapter.
|
|
113
117
|
- `dataset_rt/_dataset_rt.pyi` mirrors the Rust extension API for type checking.
|
|
114
118
|
- `dataset_rt/__init__.py` re-exports the public surface.
|
|
115
119
|
|
|
116
120
|
Rust modules:
|
|
117
121
|
|
|
118
122
|
- `types.rs`: validated IDs, config newtypes, metadata enums, samples, and `CacheError`.
|
|
119
|
-
- `storage.rs`: cache format,
|
|
123
|
+
- `storage.rs`: cache format, metadata Arrow files, indexes, shards, compression envelopes, checksums, and cache loading. `storage/manifest.rs` contains separate v2/v3 records; `storage/migration.rs` upgrades manifests while preserving resolved IDs.
|
|
120
124
|
- `writer.rs` and `writer/pipeline.rs`: source ingestion, bounded queues, serialization workers, ordered commit, progress, profiling, and publication.
|
|
121
125
|
- `dataset.rs`: Python-facing `CachedDataset`, schema checks, weight state, weight validation, and iterator construction.
|
|
122
126
|
- `runtime.rs`: Rust-owned synchronous reader pipeline with scheduler, workers, bounded result queue, and reorder buffer.
|
|
@@ -2,6 +2,111 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to DatasetRT are documented here.
|
|
4
4
|
|
|
5
|
+
## 0.4.0 - 2026-10-15
|
|
6
|
+
|
|
7
|
+
This release consolidates the 0.4 release candidates since 0.3.2.
|
|
8
|
+
|
|
9
|
+
### Highlights
|
|
10
|
+
|
|
11
|
+
- Add finite, metadata-aware PyTorch DataLoaders with persistent sampling streams,
|
|
12
|
+
automatic worker sizing, and distributed sample budgets.
|
|
13
|
+
- Add optional spawned cache-source writers and portable dataset reconstruction
|
|
14
|
+
settings that reference existing cache payloads.
|
|
15
|
+
- Reduce reader memory with shared immutable Arrow metadata and mapped indexes,
|
|
16
|
+
and bound outstanding reads together with queued and reordered results.
|
|
17
|
+
|
|
18
|
+
### Backwards Incompatible Changes
|
|
19
|
+
|
|
20
|
+
- Write new caches with v3 manifests and persistent name-derived `cache_id`
|
|
21
|
+
values. Existing v2 caches remain readable with positional IDs. Treat IDs as
|
|
22
|
+
opaque; use `CachedDataset.update_manifests(version=3)` for explicit migration,
|
|
23
|
+
constructing legacy datasets in their original path order to preserve IDs.
|
|
24
|
+
Older releases that only read v2 cannot read migrated caches.
|
|
25
|
+
- Make `CachedDataset.to_torch_dataloader()` epochs finite instead of the
|
|
26
|
+
infinite streams in early release candidates. Set `samples_per_epoch` for a
|
|
27
|
+
global sample budget, or omit it to inherit `len(dataset)` at construction.
|
|
28
|
+
Iterate the loader directly instead of externally limiting an infinite stream.
|
|
29
|
+
`batch_size` must be a positive integer; use `1` for single-sample collation.
|
|
30
|
+
- Use explicit automatic modes for `num_workers`, `native_num_workers`,
|
|
31
|
+
`persistent_workers`, and `prefetch_factor` (`"auto"`), and for seeds
|
|
32
|
+
(`"random"`). Replace previous release-candidate `None` selections accordingly.
|
|
33
|
+
Sequential loading rejects an explicit seed. Automatic persistence is enabled
|
|
34
|
+
with loading workers; automatic prefetch uses two batches per worker and is
|
|
35
|
+
disabled with zero workers. Explicit persistence or integer prefetch with
|
|
36
|
+
zero workers raises an error.
|
|
37
|
+
- Default loading processes to `multiprocessing_context="spawn"`. Use a guarded
|
|
38
|
+
application entry point and picklable callbacks. Replace release-candidate
|
|
39
|
+
`None` contexts with `"inherit"` to select Python's platform default.
|
|
40
|
+
- Default shuffled loading to replicated populations for global weighted
|
|
41
|
+
sampling; sequential loading defaults to split populations. Explicit
|
|
42
|
+
`worker_partition="split"` remains available, but equal consumer quotas can
|
|
43
|
+
approximate the global weight distribution. Replication retains private
|
|
44
|
+
weights and sampling state per consumer; 10M f64 weights require 80 MB per
|
|
45
|
+
consumer, in addition to sampling state.
|
|
46
|
+
- Divide global sample budgets equally across initialized DDP ranks, padding
|
|
47
|
+
with `drop_last=False` or discarding with `drop_last=True` to complete local
|
|
48
|
+
batches. Rank zero warns when the effective budget changes. Rank lengths and
|
|
49
|
+
reported loader lengths match emitted batches.
|
|
50
|
+
- Replace release-candidate v1 reconstruction bundles containing
|
|
51
|
+
`metadata.parquet` with v2 bundles containing `metadata.arrow`. Recreate old
|
|
52
|
+
exports with `CachedDataset.dump_config()` before calling `from_config()`.
|
|
53
|
+
|
|
54
|
+
### New Features
|
|
55
|
+
|
|
56
|
+
- Add `WriterConfig.num_processes` for bounded spawned source writers with
|
|
57
|
+
input-order outcomes; zero retains serial writing. Sources must be picklable
|
|
58
|
+
and importable. Worker death or timeout stops processing without retries.
|
|
59
|
+
- Add `CachedDataset.dump_config()` and `CachedDataset.from_config()` for
|
|
60
|
+
reconstruction settings and immutable active metadata snapshots. Loader
|
|
61
|
+
`work_dir` controls retained snapshots; the default manages temporary output.
|
|
62
|
+
- Add `python -m dataset_rt.run_benchmark` with the `benchmark` extra for local
|
|
63
|
+
rank launch or external `torchrun`, generated or existing caches, CPU/Gloo or
|
|
64
|
+
CUDA/NCCL execution, correctness checks, and JSON resource/performance reports.
|
|
65
|
+
|
|
66
|
+
### Improvements
|
|
67
|
+
|
|
68
|
+
- Resolve automatic loader counts from CPU affinity and visible container quota
|
|
69
|
+
and CPU-set limits, accounting for local ranks. Choose up to four loading
|
|
70
|
+
processes and 1–4 native threads per consumer; a one-CPU budget uses zero
|
|
71
|
+
loading processes. Set explicit counts for a reproducible worker topology.
|
|
72
|
+
- Log loader configuration and reader lifecycle through `dataset_rt.loader`.
|
|
73
|
+
`verbose=False` suppresses diagnostics while retaining the warning that
|
|
74
|
+
explicitly nonpersistent workers repeat their sampling stream each epoch.
|
|
75
|
+
- Validate worker options, finite timeouts, sample budgets, and multiprocess
|
|
76
|
+
transform picklability before exporting loader reconstruction artifacts.
|
|
77
|
+
|
|
78
|
+
### Bug Fixes
|
|
79
|
+
|
|
80
|
+
- Continue sampling across short epochs by retaining loading workers by default.
|
|
81
|
+
Explicit `persistent_workers=False` still restarts reader progress and now
|
|
82
|
+
warns about repeated samples.
|
|
83
|
+
- Preserve bounded reader memory when an early read completes late, while
|
|
84
|
+
retaining planned emission order and iterator-drop cancellation.
|
|
85
|
+
- Allow fresh native runtimes after macOS `fork` without reusing an inherited
|
|
86
|
+
thread parker; bounded channels retain FIFO delivery and backpressure.
|
|
87
|
+
- Assign whole-batch worker quotas so `len(loader)` matches emitted batches and
|
|
88
|
+
`drop_last=True` drops at most one local tail.
|
|
89
|
+
|
|
90
|
+
### Performance
|
|
91
|
+
|
|
92
|
+
- Share immutable memory-mapped Arrow metadata, selected row views, cache
|
|
93
|
+
indexes, and shuffled split permutations across consumers on one host.
|
|
94
|
+
Weights and sampling state remain private. Avoid whole-table IPC copies in
|
|
95
|
+
metadata access and updates, and stream reconstruction exports in bounded
|
|
96
|
+
chunks.
|
|
97
|
+
|
|
98
|
+
### Developers
|
|
99
|
+
|
|
100
|
+
- Add paired reader/writer benchmarks, CPU DDP acceptance examples, and
|
|
101
|
+
`just test-correctness` for fast behavioral checks with spawned workers.
|
|
102
|
+
- Preserve public imports and pickle paths while splitting Python modules;
|
|
103
|
+
enforce precise annotations and run ordinary tests with pytest-xdist.
|
|
104
|
+
|
|
105
|
+
### Validation coverage
|
|
106
|
+
|
|
107
|
+
- CPU acceptance is verified. CUDA/NCCL and physical multi-host acceptance
|
|
108
|
+
remain planned validation work on suitable hardware.
|
|
109
|
+
|
|
5
110
|
## 0.3.2 - 2026-09-28
|
|
6
111
|
|
|
7
112
|
### Added
|
|
@@ -82,6 +82,23 @@ name = "arrow-schema"
|
|
|
82
82
|
version = "54.3.1"
|
|
83
83
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
84
84
|
checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9"
|
|
85
|
+
dependencies = [
|
|
86
|
+
"bitflags 2.13.2",
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
[[package]]
|
|
90
|
+
name = "arrow-select"
|
|
91
|
+
version = "54.3.1"
|
|
92
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
93
|
+
checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619"
|
|
94
|
+
dependencies = [
|
|
95
|
+
"ahash",
|
|
96
|
+
"arrow-array",
|
|
97
|
+
"arrow-buffer",
|
|
98
|
+
"arrow-data",
|
|
99
|
+
"arrow-schema",
|
|
100
|
+
"num",
|
|
101
|
+
]
|
|
85
102
|
|
|
86
103
|
[[package]]
|
|
87
104
|
name = "autocfg"
|
|
@@ -95,6 +112,12 @@ version = "1.3.2"
|
|
|
95
112
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
96
113
|
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
|
97
114
|
|
|
115
|
+
[[package]]
|
|
116
|
+
name = "bitflags"
|
|
117
|
+
version = "2.13.2"
|
|
118
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
119
|
+
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
|
120
|
+
|
|
98
121
|
[[package]]
|
|
99
122
|
name = "block-buffer"
|
|
100
123
|
version = "0.10.4"
|
|
@@ -190,21 +213,6 @@ dependencies = [
|
|
|
190
213
|
"libc",
|
|
191
214
|
]
|
|
192
215
|
|
|
193
|
-
[[package]]
|
|
194
|
-
name = "crossbeam-channel"
|
|
195
|
-
version = "0.5.16"
|
|
196
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
197
|
-
checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
|
|
198
|
-
dependencies = [
|
|
199
|
-
"crossbeam-utils",
|
|
200
|
-
]
|
|
201
|
-
|
|
202
|
-
[[package]]
|
|
203
|
-
name = "crossbeam-utils"
|
|
204
|
-
version = "0.8.22"
|
|
205
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
206
|
-
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
|
207
|
-
|
|
208
216
|
[[package]]
|
|
209
217
|
name = "crunchy"
|
|
210
218
|
version = "0.2.4"
|
|
@@ -223,15 +231,17 @@ dependencies = [
|
|
|
223
231
|
|
|
224
232
|
[[package]]
|
|
225
233
|
name = "dataset_rt"
|
|
226
|
-
version = "0.
|
|
234
|
+
version = "0.4.0"
|
|
227
235
|
dependencies = [
|
|
228
236
|
"arrow-array",
|
|
237
|
+
"arrow-buffer",
|
|
229
238
|
"arrow-ipc",
|
|
230
239
|
"arrow-schema",
|
|
231
|
-
"
|
|
240
|
+
"arrow-select",
|
|
232
241
|
"hex",
|
|
233
242
|
"indicatif",
|
|
234
243
|
"lz4_flex",
|
|
244
|
+
"memmap2",
|
|
235
245
|
"pyo3",
|
|
236
246
|
"rand",
|
|
237
247
|
"rand_chacha",
|
|
@@ -269,7 +279,7 @@ version = "24.12.23"
|
|
|
269
279
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
270
280
|
checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096"
|
|
271
281
|
dependencies = [
|
|
272
|
-
"bitflags",
|
|
282
|
+
"bitflags 1.3.2",
|
|
273
283
|
"rustc_version",
|
|
274
284
|
]
|
|
275
285
|
|
|
@@ -447,6 +457,15 @@ version = "2.8.3"
|
|
|
447
457
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
448
458
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
449
459
|
|
|
460
|
+
[[package]]
|
|
461
|
+
name = "memmap2"
|
|
462
|
+
version = "0.9.11"
|
|
463
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
464
|
+
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
|
|
465
|
+
dependencies = [
|
|
466
|
+
"libc",
|
|
467
|
+
]
|
|
468
|
+
|
|
450
469
|
[[package]]
|
|
451
470
|
name = "num"
|
|
452
471
|
version = "0.4.3"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "dataset_rt"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
description = "Framework-independent dataset cache runtime"
|
|
6
6
|
license = "MIT"
|
|
@@ -15,13 +15,15 @@ path = "src/dataset_rt/lib.rs"
|
|
|
15
15
|
crate-type = ["cdylib"]
|
|
16
16
|
|
|
17
17
|
[dependencies]
|
|
18
|
-
arrow-array = "54"
|
|
18
|
+
arrow-array = { version = "54", features = ["ffi"] }
|
|
19
19
|
arrow-ipc = "54"
|
|
20
|
+
arrow-buffer = "54"
|
|
20
21
|
arrow-schema = "54"
|
|
21
|
-
|
|
22
|
+
arrow-select = "54"
|
|
22
23
|
hex = "0.4"
|
|
23
24
|
indicatif = "0.18"
|
|
24
25
|
lz4_flex = "0.14"
|
|
26
|
+
memmap2 = "0.9"
|
|
25
27
|
pyo3 = { version = "0.29", features = ["abi3-py310", "extension-module"] }
|
|
26
28
|
rand = "0.8"
|
|
27
29
|
rand_chacha = "0.3"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataset-rt
|
|
3
|
-
Version: 0.
|
|
4
|
-
Classifier: Development Status ::
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
7
7
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -15,11 +15,21 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
|
15
15
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
16
|
Requires-Dist: pydantic>=2
|
|
17
17
|
Requires-Dist: polars>=1
|
|
18
|
+
Requires-Dist: pyarrow>=19
|
|
19
|
+
Requires-Dist: numpy>=1.26
|
|
20
|
+
Requires-Dist: pydantic-settings>=2 ; extra == 'benchmark'
|
|
21
|
+
Requires-Dist: torch>=2.14.1 ; extra == 'benchmark'
|
|
22
|
+
Requires-Dist: psutil>=7.2.2 ; extra == 'benchmark'
|
|
18
23
|
Requires-Dist: maturin[zig]>=1.7,<2 ; extra == 'dev'
|
|
19
24
|
Requires-Dist: pytest>=8 ; extra == 'dev'
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
Requires-Dist:
|
|
25
|
+
Requires-Dist: pytest-xdist>=3.8 ; extra == 'dev'
|
|
26
|
+
Requires-Dist: ruff>=0.16.9 ; extra == 'dev'
|
|
27
|
+
Requires-Dist: pyrefly>=1.3.1 ; extra == 'dev'
|
|
22
28
|
Requires-Dist: pydantic-settings>=2 ; extra == 'dev'
|
|
29
|
+
Requires-Dist: torch>=2.14.1 ; extra == 'dev'
|
|
30
|
+
Requires-Dist: psutil>=7.2.2 ; extra == 'dev'
|
|
31
|
+
Requires-Dist: types-psutil>=7.2.2 ; extra == 'dev'
|
|
32
|
+
Provides-Extra: benchmark
|
|
23
33
|
Provides-Extra: dev
|
|
24
34
|
License-File: LICENSE
|
|
25
35
|
Summary: Framework-independent dataset cache runtime
|
|
@@ -108,7 +118,7 @@ dataset.update_metadata(balanced)
|
|
|
108
118
|
`index.bin`, shards, or manifests. Future iterators use the updated active table;
|
|
109
119
|
already-created iterators keep their snapshot.
|
|
110
120
|
|
|
111
|
-
With `ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
|
|
121
|
+
With `drt.ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
|
|
112
122
|
multinomial sampling with replacement over the active table. With
|
|
113
123
|
`shuffle=False`, it emits active rows in cyclic table order, including
|
|
114
124
|
duplicates.
|
|
@@ -120,6 +130,10 @@ length to the new table row count. Call `set_epoch_len` afterward to override it
|
|
|
120
130
|
|
|
121
131
|
## Installation
|
|
122
132
|
|
|
133
|
+
For GPU and distributed loader measurements, see the
|
|
134
|
+
[portable benchmark commands](docs/remote-validation.md):
|
|
135
|
+
`python -m dataset_rt.run_benchmark --ranks 4 --num-workers 8 --device cuda`.
|
|
136
|
+
|
|
123
137
|
DatasetRT supports Python 3.10 through 3.13.
|
|
124
138
|
|
|
125
139
|
```bash
|
|
@@ -140,12 +154,7 @@ primitive metadata.
|
|
|
140
154
|
```python
|
|
141
155
|
from pathlib import Path
|
|
142
156
|
|
|
143
|
-
|
|
144
|
-
CacheInput,
|
|
145
|
-
CacheSourcesDatasetSuccess,
|
|
146
|
-
DatasetRuntime,
|
|
147
|
-
ReaderConfig,
|
|
148
|
-
)
|
|
157
|
+
import dataset_rt as drt
|
|
149
158
|
|
|
150
159
|
|
|
151
160
|
class Images:
|
|
@@ -153,21 +162,21 @@ class Images:
|
|
|
153
162
|
|
|
154
163
|
def __iter__(self):
|
|
155
164
|
for image_id, image_bytes, label in load_my_images():
|
|
156
|
-
yield CacheInput(
|
|
165
|
+
yield drt.CacheInput(
|
|
157
166
|
data=image_bytes,
|
|
158
167
|
metadata={"image_id": image_id, "label": label},
|
|
159
168
|
)
|
|
160
169
|
|
|
161
170
|
|
|
162
|
-
runtime = DatasetRuntime(num_workers=4)
|
|
171
|
+
runtime = drt.DatasetRuntime(num_workers=4)
|
|
163
172
|
result = runtime.from_cache_sources(
|
|
164
173
|
Images(),
|
|
165
174
|
Path("cache"),
|
|
166
|
-
reader_config=ReaderConfig(seed=42, shuffle=True),
|
|
175
|
+
reader_config=drt.ReaderConfig(seed=42, shuffle=True),
|
|
167
176
|
)
|
|
168
177
|
|
|
169
178
|
match result:
|
|
170
|
-
case CacheSourcesDatasetSuccess(dataset, results):
|
|
179
|
+
case drt.CacheSourcesDatasetSuccess(dataset, results):
|
|
171
180
|
pass
|
|
172
181
|
case error:
|
|
173
182
|
raise RuntimeError(error)
|
|
@@ -77,7 +77,7 @@ dataset.update_metadata(balanced)
|
|
|
77
77
|
`index.bin`, shards, or manifests. Future iterators use the updated active table;
|
|
78
78
|
already-created iterators keep their snapshot.
|
|
79
79
|
|
|
80
|
-
With `ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
|
|
80
|
+
With `drt.ReaderConfig(shuffle=True)`, DatasetRT performs deterministic weighted
|
|
81
81
|
multinomial sampling with replacement over the active table. With
|
|
82
82
|
`shuffle=False`, it emits active rows in cyclic table order, including
|
|
83
83
|
duplicates.
|
|
@@ -89,6 +89,10 @@ length to the new table row count. Call `set_epoch_len` afterward to override it
|
|
|
89
89
|
|
|
90
90
|
## Installation
|
|
91
91
|
|
|
92
|
+
For GPU and distributed loader measurements, see the
|
|
93
|
+
[portable benchmark commands](docs/remote-validation.md):
|
|
94
|
+
`python -m dataset_rt.run_benchmark --ranks 4 --num-workers 8 --device cuda`.
|
|
95
|
+
|
|
92
96
|
DatasetRT supports Python 3.10 through 3.13.
|
|
93
97
|
|
|
94
98
|
```bash
|
|
@@ -109,12 +113,7 @@ primitive metadata.
|
|
|
109
113
|
```python
|
|
110
114
|
from pathlib import Path
|
|
111
115
|
|
|
112
|
-
|
|
113
|
-
CacheInput,
|
|
114
|
-
CacheSourcesDatasetSuccess,
|
|
115
|
-
DatasetRuntime,
|
|
116
|
-
ReaderConfig,
|
|
117
|
-
)
|
|
116
|
+
import dataset_rt as drt
|
|
118
117
|
|
|
119
118
|
|
|
120
119
|
class Images:
|
|
@@ -122,21 +121,21 @@ class Images:
|
|
|
122
121
|
|
|
123
122
|
def __iter__(self):
|
|
124
123
|
for image_id, image_bytes, label in load_my_images():
|
|
125
|
-
yield CacheInput(
|
|
124
|
+
yield drt.CacheInput(
|
|
126
125
|
data=image_bytes,
|
|
127
126
|
metadata={"image_id": image_id, "label": label},
|
|
128
127
|
)
|
|
129
128
|
|
|
130
129
|
|
|
131
|
-
runtime = DatasetRuntime(num_workers=4)
|
|
130
|
+
runtime = drt.DatasetRuntime(num_workers=4)
|
|
132
131
|
result = runtime.from_cache_sources(
|
|
133
132
|
Images(),
|
|
134
133
|
Path("cache"),
|
|
135
|
-
reader_config=ReaderConfig(seed=42, shuffle=True),
|
|
134
|
+
reader_config=drt.ReaderConfig(seed=42, shuffle=True),
|
|
136
135
|
)
|
|
137
136
|
|
|
138
137
|
match result:
|
|
139
|
-
case CacheSourcesDatasetSuccess(dataset, results):
|
|
138
|
+
case drt.CacheSourcesDatasetSuccess(dataset, results):
|
|
140
139
|
pass
|
|
141
140
|
case error:
|
|
142
141
|
raise RuntimeError(error)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from dataset_rt.api import (
|
|
2
|
+
AbsentManifestTarget,
|
|
2
3
|
CachedDataset,
|
|
3
4
|
CachedSample,
|
|
4
5
|
CacheInput,
|
|
@@ -10,7 +11,13 @@ from dataset_rt.api import (
|
|
|
10
11
|
CacheWriteResult,
|
|
11
12
|
CacheWriteSuccess,
|
|
12
13
|
DatasetRuntime,
|
|
14
|
+
Err,
|
|
15
|
+
ManifestUpdateEntry,
|
|
16
|
+
ManifestUpdateError,
|
|
17
|
+
ManifestUpdateReport,
|
|
18
|
+
Ok,
|
|
13
19
|
ReaderConfig,
|
|
20
|
+
Result,
|
|
14
21
|
ShardCompression,
|
|
15
22
|
SizedTorchIterableDataset,
|
|
16
23
|
WriterConfig,
|
|
@@ -34,4 +41,11 @@ __all__ = [
|
|
|
34
41
|
"SizedTorchIterableDataset",
|
|
35
42
|
"WriterConfig",
|
|
36
43
|
"WriterProfilerConfig",
|
|
44
|
+
"AbsentManifestTarget",
|
|
45
|
+
"Err",
|
|
46
|
+
"ManifestUpdateEntry",
|
|
47
|
+
"ManifestUpdateError",
|
|
48
|
+
"ManifestUpdateReport",
|
|
49
|
+
"Ok",
|
|
50
|
+
"Result",
|
|
37
51
|
]
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
from collections.abc import Iterator, Sequence
|
|
2
2
|
from typing import NamedTuple
|
|
3
3
|
|
|
4
|
+
from typing_extensions import CapsuleType
|
|
5
|
+
|
|
6
|
+
from dataset_rt.config import WriterConfig
|
|
7
|
+
from dataset_rt.records import CacheSource
|
|
8
|
+
|
|
4
9
|
MetadataValue = bool | int | float | str
|
|
5
10
|
|
|
6
11
|
class CacheWriteRecord(NamedTuple):
|
|
@@ -27,15 +32,23 @@ class CachedDataset:
|
|
|
27
32
|
self, cache_id: int, sample_id: int
|
|
28
33
|
) -> tuple[bytes, dict[str, MetadataValue], int, int]: ...
|
|
29
34
|
def set_epoch_len(self, epoch_len: int) -> None: ...
|
|
35
|
+
def update_manifests(
|
|
36
|
+
self, version: int
|
|
37
|
+
) -> tuple[bool, list[tuple[str, int, str]], str | None, str]: ...
|
|
30
38
|
def metadata_ipc(self) -> bytes: ...
|
|
39
|
+
def metadata_arrow(self) -> CapsuleType: ...
|
|
40
|
+
def export_metadata_arrow(self, path: str) -> int: ...
|
|
41
|
+
def update_metadata_arrow(
|
|
42
|
+
self, metadata: CapsuleType, selection: CapsuleType | None = None
|
|
43
|
+
) -> None: ...
|
|
31
44
|
def update_metadata_ipc(self, ipc: bytes) -> None: ...
|
|
32
45
|
def samples_metadata_ipc(self) -> bytes: ...
|
|
33
46
|
def set_samples_metadata_ipc(self, ipc: bytes) -> None: ...
|
|
34
47
|
|
|
35
48
|
def write_cache(
|
|
36
49
|
runtime: DatasetRuntime,
|
|
37
|
-
sources:
|
|
50
|
+
sources: CacheSource | list[CacheSource],
|
|
38
51
|
base_cache_dir: str,
|
|
39
|
-
writer_config:
|
|
52
|
+
writer_config: WriterConfig,
|
|
40
53
|
reuse_existing: bool,
|
|
41
54
|
) -> list[CacheWriteRecord]: ...
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Stable public facade retaining historical imports and pickle lookup paths."""
|
|
2
|
+
|
|
3
|
+
from dataset_rt.config import (
|
|
4
|
+
DEFAULT_SHARD_COMPRESSION as DEFAULT_SHARD_COMPRESSION,
|
|
5
|
+
)
|
|
6
|
+
from dataset_rt.config import (
|
|
7
|
+
DEFAULT_WRITER_CONFIG as DEFAULT_WRITER_CONFIG,
|
|
8
|
+
)
|
|
9
|
+
from dataset_rt.config import (
|
|
10
|
+
DEFAULT_WRITER_PROFILER_CONFIG as DEFAULT_WRITER_PROFILER_CONFIG,
|
|
11
|
+
)
|
|
12
|
+
from dataset_rt.config import (
|
|
13
|
+
CompressionAlgo as CompressionAlgo,
|
|
14
|
+
)
|
|
15
|
+
from dataset_rt.config import (
|
|
16
|
+
ReaderConfig as ReaderConfig,
|
|
17
|
+
)
|
|
18
|
+
from dataset_rt.config import (
|
|
19
|
+
ShardCompression as ShardCompression,
|
|
20
|
+
)
|
|
21
|
+
from dataset_rt.config import (
|
|
22
|
+
WriterConfig as WriterConfig,
|
|
23
|
+
)
|
|
24
|
+
from dataset_rt.config import (
|
|
25
|
+
WriterProfilerConfig as WriterProfilerConfig,
|
|
26
|
+
)
|
|
27
|
+
from dataset_rt.dataset import CachedDataset as CachedDataset
|
|
28
|
+
from dataset_rt.records import (
|
|
29
|
+
AbsentManifestTarget as AbsentManifestTarget,
|
|
30
|
+
)
|
|
31
|
+
from dataset_rt.records import (
|
|
32
|
+
BytesLike as BytesLike,
|
|
33
|
+
)
|
|
34
|
+
from dataset_rt.records import (
|
|
35
|
+
CachedSample as CachedSample,
|
|
36
|
+
)
|
|
37
|
+
from dataset_rt.records import (
|
|
38
|
+
CacheInput as CacheInput,
|
|
39
|
+
)
|
|
40
|
+
from dataset_rt.records import (
|
|
41
|
+
CacheSource as CacheSource,
|
|
42
|
+
)
|
|
43
|
+
from dataset_rt.records import (
|
|
44
|
+
CacheSourcesDatasetError as CacheSourcesDatasetError,
|
|
45
|
+
)
|
|
46
|
+
from dataset_rt.records import (
|
|
47
|
+
CacheSourcesDatasetResult as CacheSourcesDatasetResult,
|
|
48
|
+
)
|
|
49
|
+
from dataset_rt.records import (
|
|
50
|
+
CacheSourcesDatasetSuccess as CacheSourcesDatasetSuccess,
|
|
51
|
+
)
|
|
52
|
+
from dataset_rt.records import (
|
|
53
|
+
CacheWriteError as CacheWriteError,
|
|
54
|
+
)
|
|
55
|
+
from dataset_rt.records import (
|
|
56
|
+
CacheWriteResult as CacheWriteResult,
|
|
57
|
+
)
|
|
58
|
+
from dataset_rt.records import (
|
|
59
|
+
CacheWriteSuccess as CacheWriteSuccess,
|
|
60
|
+
)
|
|
61
|
+
from dataset_rt.records import (
|
|
62
|
+
Err as Err,
|
|
63
|
+
)
|
|
64
|
+
from dataset_rt.records import (
|
|
65
|
+
ManifestUpdateEntry as ManifestUpdateEntry,
|
|
66
|
+
)
|
|
67
|
+
from dataset_rt.records import (
|
|
68
|
+
ManifestUpdateError as ManifestUpdateError,
|
|
69
|
+
)
|
|
70
|
+
from dataset_rt.records import (
|
|
71
|
+
ManifestUpdateReport as ManifestUpdateReport,
|
|
72
|
+
)
|
|
73
|
+
from dataset_rt.records import (
|
|
74
|
+
Metadata as Metadata,
|
|
75
|
+
)
|
|
76
|
+
from dataset_rt.records import (
|
|
77
|
+
MetadataValue as MetadataValue,
|
|
78
|
+
)
|
|
79
|
+
from dataset_rt.records import (
|
|
80
|
+
Ok as Ok,
|
|
81
|
+
)
|
|
82
|
+
from dataset_rt.records import (
|
|
83
|
+
Result as Result,
|
|
84
|
+
)
|
|
85
|
+
from dataset_rt.records import (
|
|
86
|
+
SizedTorchIterableDataset as SizedTorchIterableDataset,
|
|
87
|
+
)
|
|
88
|
+
from dataset_rt.runtime import DatasetRuntime as DatasetRuntime
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Optional benchmark machinery; importing DatasetRT does not import its dependencies."""
|