loaderx 0.0.2__tar.gz

loaderx-0.0.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 EOELAB AI Research
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

loaderx-0.0.2/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: loaderx
+Version: 0.0.2
+Summary: Minimal data loader for Flax
+Author-email: Ben0i0d <ben0i0d@foxmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 EOELAB AI Research
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/eoeair/loaderx
+Project-URL: Documentation, https://github.com/eoeair/loaderx
+Project-URL: Source, https://github.com/eoeair/loaderx
+Project-URL: Bug Tracker, https://github.com/eoeair/loaderx/issues
+Keywords: flax,python,dataloader
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# loaderx
+Minimal data loader for Flax
+
+## Rationale for Creating mloader
+While Flax supports various data loading backends—such as PyTorch, TensorFlow, Grain, and jax_dataloader—these often come with nontrivial dependencies.
+1. Installing heavy frameworks like PyTorch or TensorFlow solely for data loading is undesirable.
+2. Grain offers a clean API but suffers from suboptimal performance in practice.
+3. jax_dataloader leverages GPU memory by default, which may lead to inefficient memory usage in certain scenarios.
+
+## Design Goals of mloader
+mloader is designed with simplicity and efficiency in mind.
+It follows a pragmatic approach—favoring low memory overhead and minimal dependencies.
+The implementation targets common use cases, with a particular focus on single-host training pipelines.
+
+## Current Limitations
+At present, mloader only supports single-host scenarios and does not yet address multi-host training setups.
+
+## How to integrate it with Flax. 
+Below is a code example.
+
+The mloader is mainly inspired by the design of Grain, so avoid using patterns like `for epoch in num_epochs`.
+
+```
+def loss_fn(model: CNN, batch):
+  logits = model(batch['data'])
+  loss = optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=batch['label']).mean()
+  return loss, logits
+
+@nnx.jit
+def train_step(model: CNN, optimizer: nnx.Optimizer, metrics: nnx.MultiMetric, batch):
+  """Train for a single step."""
+  grad_fn = nnx.value_and_grad(loss_fn, has_aux=True)
+  (loss, logits), grads = grad_fn(model, batch)
+  metrics.update(loss=loss, logits=logits, labels=batch['label'])  # In-place updates.
+  optimizer.update(grads)  # In-place updates.
+
+@nnx.jit
+def eval_step(model: CNN, metrics: nnx.MultiMetric, batch):
+  loss, logits = loss_fn(model, batch)
+  metrics.update(loss=loss, logits=logits, labels=batch['label'])  # In-place updates.
+
+@nnx.jit
+def pred_step(model: CNN, batch):
+  logits = model(batch['data'])
+  return logits.argmax(axis=1)
+```

loaderx-0.0.2/README.md

@@ -0,0 +1,46 @@
+# loaderx
+Minimal data loader for Flax
+
+## Rationale for Creating mloader
+While Flax supports various data loading backends—such as PyTorch, TensorFlow, Grain, and jax_dataloader—these often come with nontrivial dependencies.
+1. Installing heavy frameworks like PyTorch or TensorFlow solely for data loading is undesirable.
+2. Grain offers a clean API but suffers from suboptimal performance in practice.
+3. jax_dataloader leverages GPU memory by default, which may lead to inefficient memory usage in certain scenarios.
+
+## Design Goals of mloader
+mloader is designed with simplicity and efficiency in mind.
+It follows a pragmatic approach—favoring low memory overhead and minimal dependencies.
+The implementation targets common use cases, with a particular focus on single-host training pipelines.
+
+## Current Limitations
+At present, mloader only supports single-host scenarios and does not yet address multi-host training setups.
+
+## How to integrate it with Flax. 
+Below is a code example.
+
+The mloader is mainly inspired by the design of Grain, so avoid using patterns like `for epoch in num_epochs`.
+
+```
+def loss_fn(model: CNN, batch):
+  logits = model(batch['data'])
+  loss = optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=batch['label']).mean()
+  return loss, logits
+
+@nnx.jit
+def train_step(model: CNN, optimizer: nnx.Optimizer, metrics: nnx.MultiMetric, batch):
+  """Train for a single step."""
+  grad_fn = nnx.value_and_grad(loss_fn, has_aux=True)
+  (loss, logits), grads = grad_fn(model, batch)
+  metrics.update(loss=loss, logits=logits, labels=batch['label'])  # In-place updates.
+  optimizer.update(grads)  # In-place updates.
+
+@nnx.jit
+def eval_step(model: CNN, metrics: nnx.MultiMetric, batch):
+  loss, logits = loss_fn(model, batch)
+  metrics.update(loss=loss, logits=logits, labels=batch['label'])  # In-place updates.
+
+@nnx.jit
+def pred_step(model: CNN, batch):
+  logits = model(batch['data'])
+  return logits.argmax(axis=1)
+```

loaderx-0.0.2/loaderx/__init__.py

@@ -0,0 +1,4 @@
+from .dataset import Dataset
+from .dataloader import DataLoader
+
+__version__ = "0.0.2"

loaderx-0.0.2/loaderx/dataloader.py

@@ -0,0 +1,52 @@
+import numpy as np
+import threading
+from queue import Queue
+from concurrent.futures import ThreadPoolExecutor
+
+class DataLoader:
+    def __init__(self, dataset, batch_size=256, shuffle=True, prefetch=2, num_epoch=1, seed=None):
+        self.dataset = dataset
+        self.batch_size = batch_size
+        self.shuffle = shuffle
+        self.prefetch = prefetch
+        self.num_epoch = num_epoch
+        self.seed = seed
+
+        self.indices = list(range(len(dataset)))
+        self.queue = Queue(maxsize=prefetch)
+        self.stop_signal = threading.Event()
+        self.current_epoch = 0
+
+        self.thread = threading.Thread(target=self._prefetch_data)
+        self.thread.start()
+
+    def _prefetch_data(self):
+        while not self.stop_signal.is_set() and self.current_epoch < self.num_epoch:
+            if self.shuffle:
+                if self.seed is not None:
+                    np.random.seed(self.seed + self.current_epoch)
+                np.random.shuffle(self.indices)
+            for i in range(0, len(self.indices), self.batch_size):
+                batch_indices = self.indices[i:i + self.batch_size]         
+                with ThreadPoolExecutor() as executor:
+                    batch = executor.map(self.dataset.__getitem__, batch_indices)
+                batch_data, batch_labels = zip(*batch)
+                self.queue.put({'data': np.stack(batch_data), 'label': np.stack(batch_labels)})
+            self.current_epoch += 1
+        self.stop_signal.set()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.stop_signal.is_set() and self.queue.empty():
+            raise StopIteration
+        return self.queue.get()
+
+    def __del__(self):
+        try:
+            self.stop_signal.set()
+            if hasattr(self, 'thread') and self.thread is not None and self.thread.is_alive():
+                self.thread.join()
+        except Exception:
+            pass

loaderx-0.0.2/loaderx/dataset.py

@@ -0,0 +1,9 @@
+class Dataset:
+    def __init__(self):
+        pass
+
+    def __getitem__(self):
+        pass
+
+    def __len__(self):
+        pass

loaderx-0.0.2/loaderx.egg-info/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: loaderx
+Version: 0.0.2
+Summary: Minimal data loader for Flax
+Author-email: Ben0i0d <ben0i0d@foxmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 EOELAB AI Research
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/eoeair/loaderx
+Project-URL: Documentation, https://github.com/eoeair/loaderx
+Project-URL: Source, https://github.com/eoeair/loaderx
+Project-URL: Bug Tracker, https://github.com/eoeair/loaderx/issues
+Keywords: flax,python,dataloader
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# loaderx
+Minimal data loader for Flax
+
+## Rationale for Creating mloader
+While Flax supports various data loading backends—such as PyTorch, TensorFlow, Grain, and jax_dataloader—these often come with nontrivial dependencies.
+1. Installing heavy frameworks like PyTorch or TensorFlow solely for data loading is undesirable.
+2. Grain offers a clean API but suffers from suboptimal performance in practice.
+3. jax_dataloader leverages GPU memory by default, which may lead to inefficient memory usage in certain scenarios.
+
+## Design Goals of mloader
+mloader is designed with simplicity and efficiency in mind.
+It follows a pragmatic approach—favoring low memory overhead and minimal dependencies.
+The implementation targets common use cases, with a particular focus on single-host training pipelines.
+
+## Current Limitations
+At present, mloader only supports single-host scenarios and does not yet address multi-host training setups.
+
+## How to integrate it with Flax. 
+Below is a code example.
+
+The mloader is mainly inspired by the design of Grain, so avoid using patterns like `for epoch in num_epochs`.
+
+```
+def loss_fn(model: CNN, batch):
+  logits = model(batch['data'])
+  loss = optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=batch['label']).mean()
+  return loss, logits
+
+@nnx.jit
+def train_step(model: CNN, optimizer: nnx.Optimizer, metrics: nnx.MultiMetric, batch):
+  """Train for a single step."""
+  grad_fn = nnx.value_and_grad(loss_fn, has_aux=True)
+  (loss, logits), grads = grad_fn(model, batch)
+  metrics.update(loss=loss, logits=logits, labels=batch['label'])  # In-place updates.
+  optimizer.update(grads)  # In-place updates.
+
+@nnx.jit
+def eval_step(model: CNN, metrics: nnx.MultiMetric, batch):
+  loss, logits = loss_fn(model, batch)
+  metrics.update(loss=loss, logits=logits, labels=batch['label'])  # In-place updates.
+
+@nnx.jit
+def pred_step(model: CNN, batch):
+  logits = model(batch['data'])
+  return logits.argmax(axis=1)
+```

loaderx-0.0.2/loaderx.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+LICENSE
+README.md
+pyproject.toml
+loaderx/__init__.py
+loaderx/dataloader.py
+loaderx/dataset.py
+loaderx.egg-info/PKG-INFO
+loaderx.egg-info/SOURCES.txt
+loaderx.egg-info/dependency_links.txt
+loaderx.egg-info/top_level.txt

loaderx-0.0.2/loaderx.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

loaderx-0.0.2/loaderx.egg-info/top_level.txt

@@ -0,0 +1 @@
+loaderx

loaderx-0.0.2/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "loaderx"
+dynamic = ["version"]
+description = "Minimal data loader for Flax"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [
+    {name = "Ben0i0d", email = "ben0i0d@foxmail.com"},
+]
+license = {file = "LICENSE"}
+keywords = ["flax", "python", "dataloader"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+]
+dependencies = []
+
+[tool.setuptools.dynamic]
+version = {attr = "loaderx.__version__"}
+
+[project.urls]
+"Homepage" = "https://github.com/eoeair/loaderx"
+"Documentation" = "https://github.com/eoeair/loaderx"
+"Source" = "https://github.com/eoeair/loaderx"
+"Bug Tracker" = "https://github.com/eoeair/loaderx/issues"

loaderx-0.0.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+