dataset-rt 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. dataset_rt-0.1.3/.github/workflows/ci.yml +51 -0
  2. dataset_rt-0.1.3/.gitignore +17 -0
  3. dataset_rt-0.1.3/.python-version +1 -0
  4. dataset_rt-0.1.3/CHANGELOG.md +39 -0
  5. dataset_rt-0.1.3/Cargo.lock +930 -0
  6. dataset_rt-0.1.3/Cargo.toml +29 -0
  7. dataset_rt-0.1.3/LICENSE +21 -0
  8. dataset_rt-0.1.3/PKG-INFO +211 -0
  9. dataset_rt-0.1.3/README.md +179 -0
  10. dataset_rt-0.1.3/dataset_rt/__init__.py +23 -0
  11. dataset_rt-0.1.3/dataset_rt/_dataset_rt.pyi +27 -0
  12. dataset_rt-0.1.3/dataset_rt/api.py +297 -0
  13. dataset_rt-0.1.3/dataset_rt/py.typed +1 -0
  14. dataset_rt-0.1.3/dataset_rt_spec.md +312 -0
  15. dataset_rt-0.1.3/docs/architecture.md +78 -0
  16. dataset_rt-0.1.3/docs/build.md +60 -0
  17. dataset_rt-0.1.3/docs/determinism.md +50 -0
  18. dataset_rt-0.1.3/docs/development.md +55 -0
  19. dataset_rt-0.1.3/docs/python-api.md +189 -0
  20. dataset_rt-0.1.3/docs/runtime.md +77 -0
  21. dataset_rt-0.1.3/docs/serialization.md +32 -0
  22. dataset_rt-0.1.3/docs/storage-format.md +81 -0
  23. dataset_rt-0.1.3/justfile +55 -0
  24. dataset_rt-0.1.3/pyproject.toml +62 -0
  25. dataset_rt-0.1.3/rust-toolchain.toml +2 -0
  26. dataset_rt-0.1.3/scripts/bench_smoke.py +62 -0
  27. dataset_rt-0.1.3/scripts/build_local.py +182 -0
  28. dataset_rt-0.1.3/src/dataset_rt/dataset.rs +432 -0
  29. dataset_rt-0.1.3/src/dataset_rt/lib.rs +45 -0
  30. dataset_rt-0.1.3/src/dataset_rt/runtime.rs +219 -0
  31. dataset_rt-0.1.3/src/dataset_rt/sampling.rs +49 -0
  32. dataset_rt-0.1.3/src/dataset_rt/storage.rs +626 -0
  33. dataset_rt-0.1.3/src/dataset_rt/types.rs +245 -0
  34. dataset_rt-0.1.3/src/dataset_rt/writer.rs +458 -0
  35. dataset_rt-0.1.3/tests/integrations/test_pytorch.py +55 -0
  36. dataset_rt-0.1.3/tests/test_dataset_rt.py +254 -0
  37. dataset_rt-0.1.3/tests/test_integrity.py +96 -0
  38. dataset_rt-0.1.3/uv.lock +465 -0
dataset_rt-0.1.3/.github/workflows/ci.yml ADDED
@@ -0,0 +1,51 @@
1
+ name: CI
2
+
3
+ on:
4
+ pull_request:
5
+ push:
6
+ branches:
7
+ - main
8
+
9
+ jobs:
10
+ checks:
11
+ name: Python 3.11 and Rust stable
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - name: Checkout
16
+ uses: actions/checkout@v4
17
+
18
+ - name: Install Rust stable
19
+ uses: dtolnay/rust-toolchain@stable
20
+
21
+ - name: Install uv
22
+ uses: astral-sh/setup-uv@v5
23
+ with:
24
+ enable-cache: true
25
+
26
+ - name: Set up Python 3.11
27
+ run: uv python install 3.11
28
+
29
+ - name: Sync dependencies
30
+ run: uv sync --python 3.11 --extra dev
31
+
32
+ - name: Rust tests
33
+ run: cargo test
34
+
35
+ - name: Clippy
36
+ run: cargo clippy --all-targets -- -D warnings
37
+
38
+ - name: Build editable extension
39
+ run: uv run --python 3.11 --extra dev maturin develop
40
+
41
+ - name: Ruff format
42
+ run: uv run --python 3.11 --extra dev ruff format --check dataset_rt tests scripts
43
+
44
+ - name: Ruff lint
45
+ run: uv run --python 3.11 --extra dev ruff check dataset_rt tests scripts
46
+
47
+ - name: Pyrefly
48
+ run: uv run --python 3.11 --extra dev pyrefly check
49
+
50
+ - name: Pytest
51
+ run: uv run --python 3.11 --extra dev pytest
dataset_rt-0.1.3/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ /target/
2
+ /.venv/
3
+ /dist/
4
+ /wheelhouse/
5
+ *.egg-info/
6
+ **/__pycache__/
7
+ /.pytest_cache/
8
+ /.ruff_cache/
9
+ /.pyrefly/
10
+ /.mypy_cache/
11
+ /.coverage
12
+ /htmlcov/
13
+ .DS_Store
14
+ *.so
15
+ *.pyd
16
+ *.pyc
17
+ *.core
dataset_rt-0.1.3/.python-version ADDED
@@ -0,0 +1 @@
1
+ 3.11
dataset_rt-0.1.3/CHANGELOG.md ADDED
@@ -0,0 +1,39 @@
1
+ # Changelog
2
+
3
+ All notable changes to DatasetRT are documented here.
4
+
5
+ ## 0.1.3 - 2026-07-28
6
+
7
+ ### Changed
8
+
9
+ - Bumped the package patch version for a fresh PyPI artifact set.
10
+
11
+ ## 0.1.2 - 2026-07-28
12
+
13
+ ### Changed
14
+
15
+ - Rebuilt artifact packaging with `cp310-abi3` wheels for Python 3.10 through 3.13.
16
+ - Added local Linux `x86_64` and `aarch64` wheel builds through `maturin --zig`.
17
+ - Clarified local artifact building with `just build-all`.
18
+
19
+ ## 0.1.0 - 2026-07-28
20
+
21
+ ### Added
22
+
23
+ - Rust-backed immutable cache storage with manifests, checksums, Arrow metadata, binary indexes, and payload shards.
24
+ - Thin Python API with `CacheSource`, `CacheInput`, `CachedDataset`, `ReaderConfig`, `WriterConfig`, and `ShardCompression`.
25
+ - `CachedDataset.from_cache_sources` factory that creates missing caches, reuses valid existing caches, and loads the dataset.
26
+ - Deterministic weighted sampling with Polars `weight_table` and `set_weight_table`.
27
+ - Reader configuration for `prefetch_size`, `num_workers`, and `shuffle`.
28
+ - Rust-owned writer prefetching, worker threads, ordered commits, and multi-source cache writes.
29
+ - Optional sized PyTorch `IterableDataset` adapter through `to_torch_iterable_dataset`.
30
+ - Stable ABI wheels (`cp310-abi3`) for Python 3.10 through 3.13.
31
+ - Local artifact build tooling for macOS arm64, macOS x86_64, Linux x86_64, Linux aarch64, and sdist artifacts.
32
+ - GitHub CI for checks, plus `just` project commands for common workflows.
33
+
34
+ ### Limits
35
+
36
+ - Payloads are bytes-like only: `bytes`, `bytearray`, or `memoryview`.
37
+ - Metadata values are primitive only: `bool`, `int`, `float`, or `str`.
38
+ - Shard compression is intentionally limited to `ShardCompression(algo="none", ratio=1.0)`.
39
+ - Domain decoding, such as JPEG/tensor/object decoding, stays in Python.