loaderx 2.3.3__tar.gz → 2.3.5__tar.gz

{loaderx-2.3.3/loaderx.egg-info → loaderx-2.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loaderx
-Version: 2.3.3
+Version: 2.3.5
 Summary: Rebuildable high-performance ordered record containers
 Author-email: Ben0i0d <ben0i0d@foxmail.com>
 License-Expression: MIT
@@ -21,6 +21,8 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: cffi
 Requires-Dist: msgpack
+Provides-Extra: converter
+Requires-Dist: datasets[vision]>=2.19; extra == "converter"
 Dynamic: license-file
 
 # Loaderx
@@ -73,9 +75,9 @@ from loaderx.dataloader import DataLoader
 
 data = np.load('data.npy', mmap_mode='r')
 label = np.load('label.npy', mmap_mode='r')
-with Dense.create('train_data', data.dtype, data.shape[1:]) as ds:
+with Dense.create('train_data', data.dtype, data.shape[1:], codec='zstd') as ds:
     ds.append(data)
-with Dense.create('train_label', label.dtype, label.shape[1:]) as ds:
+with Dense.create('train_label', label.dtype, label.shape[1:], codec='zstd') as ds:
     ds.append(label)
 
 data_store = Dense.open('train_data')
@@ -100,13 +102,70 @@ at the same indices, so record ``i`` lines up across them.  The ``transform``
 callback is the collate step — reshape, cast, stack — where ``values`` is the
 plain dense batch ready for the model.
 
+### Offline Hugging Face conversion
+
+The optional converter uses Hugging Face Datasets for remote discovery,
+download, caching, revision handling and source-format decoding. It then writes
+explicit typed Zrecord streams, so training needs neither `datasets` nor Arrow:
+
+```bash
+pip install 'loaderx[converter]'
+```
+
+```python
+from loaderx.converter import convert, huggingface
+
+dataset = huggingface("ylecun/mnist", revision="main", token=None)
+convert(
+    dataset,
+    "mnist",
+    codec="zstd",
+    batch_size=256,
+)
+```
+
+`huggingface` resolves the requested branch, tag or commit to an immutable
+snapshot, then downloads and returns the repository's DatasetDict. Repositories
+without one default config require `config=...`. `convert` is the independent
+persistence stage: it accepts that DatasetDict or one already normalized
+Dataset. `batch_size` bounds records per Python batch; actual bytes also depend
+on record sizes and Ragged packing.
+For private or gated repositories, pass `token="hf_..."`; `None` uses the
+standard HF environment, local login, or anonymous access.
+Static or regularly batched numeric arrays become Dense stores, while dynamic
+arrays become Ragged stores; source dtype and record shape are preserved.
+
+Zrecord persists numeric tensors rather than Python objects. A standard HF
+feature that cannot be represented losslessly fails with its column name;
+conversion does not filter, pad, tokenize, reshape, or run a transform stage.
+Arrow dtypes, including float64 and uint64, are preserved. Only the current
+batch is formatted; `with_format` never materializes the complete dataset.
+
+The result groups aligned streams under one published root:
+
+```text
+mnist/
+  train/
+    image/
+    label/
+  test/
+    image/
+    label/
+```
+
+Conversion happens under a temporary sibling directory. The root is renamed
+into place only after all stores close and their record counts agree; a failed
+build that raises a Python exception is discarded. A process kill may leave the
+hidden temporary sibling for manual cleanup. No collection manifest is added;
+the output is only split directories containing ordinary column Zrecord stores.
+
 ### Creating a dense store
 ```python
 import numpy as np
 from loaderx.zrecord import Dense
 
 data = np.load('data.npy', mmap_mode='r')
-with Dense.create('train_data', data.dtype, data.shape[1:]) as ds:
+with Dense.create('train_data', data.dtype, data.shape[1:], codec='zstd') as ds:
     ds.append(data)
 ```
 One record per slice along axis 0; a 1-D array (the usual shape of a label set)
@@ -132,7 +191,7 @@ engine consumes the trusted Dense stride or Ragged offsets. Append inputs are
 strictly NumPy arrays: Dense takes one batched ndarray and Ragged takes an
 iterable of ndarrays. Raw bytes and pre-encoded images are made explicit with
 ``np.frombuffer(raw, dtype=np.uint8)`` and stored in a
-``Ragged.create(path, dtype=np.uint8, ndim=1)`` rather than creating a second
+``Ragged.create(path, dtype=np.uint8, ndim=1, codec="zstd")`` rather than creating a second
 public storage API.
 
 ### Records
@@ -147,7 +206,7 @@ import numpy as np
 from loaderx.zrecord import Dense
 
 data = np.arange(64, dtype=np.float32).reshape(8, 2, 4)
-with Dense.create('data', data.dtype, data.shape[1:]) as ds:
+with Dense.create('data', data.dtype, data.shape[1:], codec='zstd') as ds:
     ds.append(data)
 ds = Dense.open('data')
 ds[0, 5, 2]                          # (3, 2, 4) — shape from the persisted schema
@@ -173,7 +232,7 @@ batch is the model's call — a plain numpy loop, wherever you need it:
 from loaderx.zrecord import Ragged
 
 seqs = [np.arange(L, dtype=np.int32) for L in (3, 1, 4, 1, 5)]
-with Ragged.create('tokens', np.int32, ndim=1) as rs:
+with Ragged.create('tokens', np.int32, ndim=1, codec='zstd') as rs:
     rs.append(seqs)                       # dtype/rank fixed; lengths remain per-record
 rs = Ragged.open('tokens')
 
@@ -260,8 +319,10 @@ warmup belongs outside build or loader benchmark timing.
 ### Creating containers
 
 ``Dense.create`` and ``Ragged.create`` return append-only ordered-sequence
-builders. ``append`` is explicit — one input batch extends the logical sequence
-without exposing native physical completion order.
+builders. The ``codec`` keyword is required: callers must explicitly select
+``"raw"``, ``"zstd"``, or ``"zstd_dict"``. ``append`` is explicit — one input
+batch extends the logical sequence without exposing native physical completion
+order.
 Dense append is synchronous and borrows an already-contiguous ndarray without a
 snapshot copy. Ragged append consumes its iterable once into one owned packed
 buffer, then completes the native append before returning. Nothing is inferred.
@@ -269,12 +330,13 @@ buffer, then completes the native append before returning. Nothing is inferred.
 ```python
 from loaderx.zrecord import Dense, Ragged
 
-ds = Dense.create('mnist/x', dtype=np.uint8, item_shape=(28, 28), data_shards=4)
+ds = Dense.create('mnist/x', dtype=np.uint8, item_shape=(28, 28),
+                  codec='zstd', data_shards=4)
 ds.append(images[i:i + 1024])        # synchronous native batch; returns None
 ds.append(single_image[None])        # one sample is batch_size 1 — add the axis yourself
 ds.close()                            # publish before opening
 
-tok = Ragged.create('tokens', dtype=np.int32, ndim=1)
+tok = Ragged.create('tokens', dtype=np.int32, ndim=1, codec='zstd')
 tok.append([seq_a, seq_b, seq_c])
 tok.close()
 
@@ -315,7 +377,7 @@ published without one.
 ### Codec notes
 
 `"zstd"` compresses each record independently with plain zstd (level 3). Use it
-for general-purpose compression — it is fast and the default.
+for general-purpose compression; it is fast and must be selected explicitly.
 
 `"zstd_dict"` trains a shared dictionary on a sample of the data before writing
 any record, then compresses every record against it at level 15. The dictionary
@@ -409,11 +471,11 @@ from loaderx.zrecord import Dense, Ragged
 from loaderx.dataloader import DataLoader
 
 root = "xsub/train"
-with Dense.create(root + "/joint", joint.dtype, joint.shape[1:]) as s:
+with Dense.create(root + "/joint", joint.dtype, joint.shape[1:], codec="zstd") as s:
     s.append(joint)
-with Dense.create(root + "/label", label.dtype, label.shape[1:]) as s:
+with Dense.create(root + "/label", label.dtype, label.shape[1:], codec="zstd") as s:
     s.append(label)
-with Ragged.create(root + "/token", np.int32, ndim=1) as s:
+with Ragged.create(root + "/token", np.int32, ndim=1, codec="zstd") as s:
     s.append(seqs)
 
 streams = {
@@ -895,57 +957,13 @@ The loader rows used Numba 0.67.0, Torch 2.13.0 and Grain 0.2.18; all benchmark
 are pinned in `scripts/requirements-bench.txt`. Temporary stores used the ordinary
 disk-backed `/tmp` filesystem, not `/dev/shm`.
 
-## Real-data verification: NTU RGB-D skeletons
-
-The vision tables above are synthetic; the token tables use real WikiText-103.
-As a separate historical ground-truth check,
-loaderx was run end to end on **NTU RGB-D skeleton data** — 114,480 raw `.skeleton` files,
-120 action classes, 25 joints — processed into the ST-GCN `N C T V M` layout
-(per-sample `(3, 300, 25, 2)` float32) for the `xsub`/`xview` protocols. The
-`.npy` outputs of the standard preprocessing pipeline were treated as ground
-truth. This verification was not rerun with the synthetic benchmarks above;
-its throughput is retained as a separate historical 12-core result.
-
-**Correctness — the read path is bit-exact against the ground truth:**
-
-* Full scan of all **228,356 records** (joint float32 + label int64, all four
-  splits) through `Dense`: byte-for-byte identical to the reference npy.
-* A `DataLoader` over joint + label + an index stream, run under all three
-  sampler modes (`sequential`, `iid`, `cyclic`): every received batch is
-  bit-exact to the ground truth at its own declared indices, and the streams
-  stay index-aligned.
-* Sampler semantics hold on real index spaces: `sequential` walks in order,
-  `cyclic` draws a full cycle without replacement, `iid` is deterministic per
-  seed.
-
-**Storage — zstd on this data:**
-
-| store                 | on disk | ratio |
-|-----------------------|---------|-------|
-| npy (raw float32)     | 6.4 GB  | 1.00x |
-| zrecord raw           | 6.86 GB | 1.00x |
-| zrecord zstd          | 0.79 GB | 8.66x |
-| zrecord zstd_dict     | 0.75 GB | 9.13x |
-
-Sizes above are for one split (`xview/val`, 38,132 records); across all four
-splits the zstd `joint` stores total 4.97 GB against 41 GB of raw npy (~8x).
-
-**Throughput** (180 KB per record, warm page cache, 12 physical cores):
-
-| path                                | throughput |
-|-------------------------------------|-----------|
-| random-batch gather, zstd store     | 4.1–4.5 GiB/s |
-| same, npy-mmap fancy indexing       | 0.6–1.3 GiB/s |
-| DataLoader, 4 prefetch threads      | 5.7–6.6 GiB/s (123–144 batches/s) |
-
-zstd decompression reads ~8x fewer bytes than raw storage, so the compressed
-store gathers *faster* than the raw one (zstd 4587 MiB/s vs raw 1792 MiB/s on
-the same split).
-
-**The npy intermediate is optional.** Parse the skeleton files in parallel and
-feed each fixed-shape ndarray produced by the parser directly to
-`Dense.append`. This writes the store in one pass with no npy staging or
-second read; zrecord bounds its compression working memory independently.
+## Real-data verification
+
+Dataset-specific preprocessing and verification remain in the
+**[DataPipe](https://codeberg.org/eoelab/DataPipe)** repository. The built-in
+converter covers standardized Hugging Face datasets; DataPipe handles sources
+without a common remote protocol and implements derived modalities as loader
+transforms without NumPy dump intermediates.
 
 ## Current Limitations
 * Single-host only; multi-host training is not supported.

{loaderx-2.3.3 → loaderx-2.3.5}/README.md

@@ -48,9 +48,9 @@ from loaderx.dataloader import DataLoader
 
 data = np.load('data.npy', mmap_mode='r')
 label = np.load('label.npy', mmap_mode='r')
-with Dense.create('train_data', data.dtype, data.shape[1:]) as ds:
+with Dense.create('train_data', data.dtype, data.shape[1:], codec='zstd') as ds:
     ds.append(data)
-with Dense.create('train_label', label.dtype, label.shape[1:]) as ds:
+with Dense.create('train_label', label.dtype, label.shape[1:], codec='zstd') as ds:
     ds.append(label)
 
 data_store = Dense.open('train_data')
@@ -75,13 +75,70 @@ at the same indices, so record ``i`` lines up across them.  The ``transform``
 callback is the collate step — reshape, cast, stack — where ``values`` is the
 plain dense batch ready for the model.
 
+### Offline Hugging Face conversion
+
+The optional converter uses Hugging Face Datasets for remote discovery,
+download, caching, revision handling and source-format decoding. It then writes
+explicit typed Zrecord streams, so training needs neither `datasets` nor Arrow:
+
+```bash
+pip install 'loaderx[converter]'
+```
+
+```python
+from loaderx.converter import convert, huggingface
+
+dataset = huggingface("ylecun/mnist", revision="main", token=None)
+convert(
+    dataset,
+    "mnist",
+    codec="zstd",
+    batch_size=256,
+)
+```
+
+`huggingface` resolves the requested branch, tag or commit to an immutable
+snapshot, then downloads and returns the repository's DatasetDict. Repositories
+without one default config require `config=...`. `convert` is the independent
+persistence stage: it accepts that DatasetDict or one already normalized
+Dataset. `batch_size` bounds records per Python batch; actual bytes also depend
+on record sizes and Ragged packing.
+For private or gated repositories, pass `token="hf_..."`; `None` uses the
+standard HF environment, local login, or anonymous access.
+Static or regularly batched numeric arrays become Dense stores, while dynamic
+arrays become Ragged stores; source dtype and record shape are preserved.
+
+Zrecord persists numeric tensors rather than Python objects. A standard HF
+feature that cannot be represented losslessly fails with its column name;
+conversion does not filter, pad, tokenize, reshape, or run a transform stage.
+Arrow dtypes, including float64 and uint64, are preserved. Only the current
+batch is formatted; `with_format` never materializes the complete dataset.
+
+The result groups aligned streams under one published root:
+
+```text
+mnist/
+  train/
+    image/
+    label/
+  test/
+    image/
+    label/
+```
+
+Conversion happens under a temporary sibling directory. The root is renamed
+into place only after all stores close and their record counts agree; a failed
+build that raises a Python exception is discarded. A process kill may leave the
+hidden temporary sibling for manual cleanup. No collection manifest is added;
+the output is only split directories containing ordinary column Zrecord stores.
+
 ### Creating a dense store
 ```python
 import numpy as np
 from loaderx.zrecord import Dense
 
 data = np.load('data.npy', mmap_mode='r')
-with Dense.create('train_data', data.dtype, data.shape[1:]) as ds:
+with Dense.create('train_data', data.dtype, data.shape[1:], codec='zstd') as ds:
     ds.append(data)
 ```
 One record per slice along axis 0; a 1-D array (the usual shape of a label set)
@@ -107,7 +164,7 @@ engine consumes the trusted Dense stride or Ragged offsets. Append inputs are
 strictly NumPy arrays: Dense takes one batched ndarray and Ragged takes an
 iterable of ndarrays. Raw bytes and pre-encoded images are made explicit with
 ``np.frombuffer(raw, dtype=np.uint8)`` and stored in a
-``Ragged.create(path, dtype=np.uint8, ndim=1)`` rather than creating a second
+``Ragged.create(path, dtype=np.uint8, ndim=1, codec="zstd")`` rather than creating a second
 public storage API.
 
 ### Records
@@ -122,7 +179,7 @@ import numpy as np
 from loaderx.zrecord import Dense
 
 data = np.arange(64, dtype=np.float32).reshape(8, 2, 4)
-with Dense.create('data', data.dtype, data.shape[1:]) as ds:
+with Dense.create('data', data.dtype, data.shape[1:], codec='zstd') as ds:
     ds.append(data)
 ds = Dense.open('data')
 ds[0, 5, 2]                          # (3, 2, 4) — shape from the persisted schema
@@ -148,7 +205,7 @@ batch is the model's call — a plain numpy loop, wherever you need it:
 from loaderx.zrecord import Ragged
 
 seqs = [np.arange(L, dtype=np.int32) for L in (3, 1, 4, 1, 5)]
-with Ragged.create('tokens', np.int32, ndim=1) as rs:
+with Ragged.create('tokens', np.int32, ndim=1, codec='zstd') as rs:
     rs.append(seqs)                       # dtype/rank fixed; lengths remain per-record
 rs = Ragged.open('tokens')
 
@@ -235,8 +292,10 @@ warmup belongs outside build or loader benchmark timing.
 ### Creating containers
 
 ``Dense.create`` and ``Ragged.create`` return append-only ordered-sequence
-builders. ``append`` is explicit — one input batch extends the logical sequence
-without exposing native physical completion order.
+builders. The ``codec`` keyword is required: callers must explicitly select
+``"raw"``, ``"zstd"``, or ``"zstd_dict"``. ``append`` is explicit — one input
+batch extends the logical sequence without exposing native physical completion
+order.
 Dense append is synchronous and borrows an already-contiguous ndarray without a
 snapshot copy. Ragged append consumes its iterable once into one owned packed
 buffer, then completes the native append before returning. Nothing is inferred.
@@ -244,12 +303,13 @@ buffer, then completes the native append before returning. Nothing is inferred.
 ```python
 from loaderx.zrecord import Dense, Ragged
 
-ds = Dense.create('mnist/x', dtype=np.uint8, item_shape=(28, 28), data_shards=4)
+ds = Dense.create('mnist/x', dtype=np.uint8, item_shape=(28, 28),
+                  codec='zstd', data_shards=4)
 ds.append(images[i:i + 1024])        # synchronous native batch; returns None
 ds.append(single_image[None])        # one sample is batch_size 1 — add the axis yourself
 ds.close()                            # publish before opening
 
-tok = Ragged.create('tokens', dtype=np.int32, ndim=1)
+tok = Ragged.create('tokens', dtype=np.int32, ndim=1, codec='zstd')
 tok.append([seq_a, seq_b, seq_c])
 tok.close()
 
@@ -290,7 +350,7 @@ published without one.
 ### Codec notes
 
 `"zstd"` compresses each record independently with plain zstd (level 3). Use it
-for general-purpose compression — it is fast and the default.
+for general-purpose compression; it is fast and must be selected explicitly.
 
 `"zstd_dict"` trains a shared dictionary on a sample of the data before writing
 any record, then compresses every record against it at level 15. The dictionary
@@ -384,11 +444,11 @@ from loaderx.zrecord import Dense, Ragged
 from loaderx.dataloader import DataLoader
 
 root = "xsub/train"
-with Dense.create(root + "/joint", joint.dtype, joint.shape[1:]) as s:
+with Dense.create(root + "/joint", joint.dtype, joint.shape[1:], codec="zstd") as s:
     s.append(joint)
-with Dense.create(root + "/label", label.dtype, label.shape[1:]) as s:
+with Dense.create(root + "/label", label.dtype, label.shape[1:], codec="zstd") as s:
     s.append(label)
-with Ragged.create(root + "/token", np.int32, ndim=1) as s:
+with Ragged.create(root + "/token", np.int32, ndim=1, codec="zstd") as s:
     s.append(seqs)
 
 streams = {
@@ -870,57 +930,13 @@ The loader rows used Numba 0.67.0, Torch 2.13.0 and Grain 0.2.18; all benchmark
 are pinned in `scripts/requirements-bench.txt`. Temporary stores used the ordinary
 disk-backed `/tmp` filesystem, not `/dev/shm`.
 
-## Real-data verification: NTU RGB-D skeletons
-
-The vision tables above are synthetic; the token tables use real WikiText-103.
-As a separate historical ground-truth check,
-loaderx was run end to end on **NTU RGB-D skeleton data** — 114,480 raw `.skeleton` files,
-120 action classes, 25 joints — processed into the ST-GCN `N C T V M` layout
-(per-sample `(3, 300, 25, 2)` float32) for the `xsub`/`xview` protocols. The
-`.npy` outputs of the standard preprocessing pipeline were treated as ground
-truth. This verification was not rerun with the synthetic benchmarks above;
-its throughput is retained as a separate historical 12-core result.
-
-**Correctness — the read path is bit-exact against the ground truth:**
-
-* Full scan of all **228,356 records** (joint float32 + label int64, all four
-  splits) through `Dense`: byte-for-byte identical to the reference npy.
-* A `DataLoader` over joint + label + an index stream, run under all three
-  sampler modes (`sequential`, `iid`, `cyclic`): every received batch is
-  bit-exact to the ground truth at its own declared indices, and the streams
-  stay index-aligned.
-* Sampler semantics hold on real index spaces: `sequential` walks in order,
-  `cyclic` draws a full cycle without replacement, `iid` is deterministic per
-  seed.
-
-**Storage — zstd on this data:**
-
-| store                 | on disk | ratio |
-|-----------------------|---------|-------|
-| npy (raw float32)     | 6.4 GB  | 1.00x |
-| zrecord raw           | 6.86 GB | 1.00x |
-| zrecord zstd          | 0.79 GB | 8.66x |
-| zrecord zstd_dict     | 0.75 GB | 9.13x |
-
-Sizes above are for one split (`xview/val`, 38,132 records); across all four
-splits the zstd `joint` stores total 4.97 GB against 41 GB of raw npy (~8x).
-
-**Throughput** (180 KB per record, warm page cache, 12 physical cores):
-
-| path                                | throughput |
-|-------------------------------------|-----------|
-| random-batch gather, zstd store     | 4.1–4.5 GiB/s |
-| same, npy-mmap fancy indexing       | 0.6–1.3 GiB/s |
-| DataLoader, 4 prefetch threads      | 5.7–6.6 GiB/s (123–144 batches/s) |
-
-zstd decompression reads ~8x fewer bytes than raw storage, so the compressed
-store gathers *faster* than the raw one (zstd 4587 MiB/s vs raw 1792 MiB/s on
-the same split).
-
-**The npy intermediate is optional.** Parse the skeleton files in parallel and
-feed each fixed-shape ndarray produced by the parser directly to
-`Dense.append`. This writes the store in one pass with no npy staging or
-second read; zrecord bounds its compression working memory independently.
+## Real-data verification
+
+Dataset-specific preprocessing and verification remain in the
+**[DataPipe](https://codeberg.org/eoelab/DataPipe)** repository. The built-in
+converter covers standardized Hugging Face datasets; DataPipe handles sources
+without a common remote protocol and implements derived modalities as loader
+transforms without NumPy dump intermediates.
 
 ## Current Limitations
 * Single-host only; multi-host training is not supported.

{loaderx-2.3.3 → loaderx-2.3.5}/loaderx/__init__.py

@@ -4,11 +4,13 @@ Public API is imported from its layer, not flattened here, so the abstraction
 boundaries stay visible:
 
     from loaderx.utils      import train_dict
+    from loaderx.converter  import convert, huggingface
     from loaderx.zrecord    import Dense, Ragged, ZrecordError
     from loaderx.dataloader  import DataLoader
     from loaderx.zsampler    import Sampler
 
-``utils`` holds the manual, standalone ``train_dict`` step. ``zrecord`` exposes
+``utils`` holds the manual, standalone ``train_dict`` step. ``converter`` owns
+offline remote ingestion into completed Zrecord streams. ``zrecord`` exposes
 ``Dense`` and ``Ragged`` are rebuildable ordered containers: ``create`` extends
 the sequence in append order, ``open`` is read-only, and ``close`` is the sole
 public publication boundary.
@@ -16,4 +18,4 @@ public publication boundary.
 step-based sampler + prefetch pipeline over them.
 """
 
-__version__ = "2.3.3"
+__version__ = "2.3.5"