loaderx 0.2.1__tar.gz → 0.3.0__tar.gz

{loaderx-0.2.1 → loaderx-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loaderx
-Version: 0.2.1
+Version: 0.3.0
 Summary: A record-based data runtime, focused on delivering extreme throughput and low latency
 Author-email: Ben0i0d <ben0i0d@foxmail.com>
 License: MIT License
@@ -34,7 +34,7 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.10
+Requires-Python: ==3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
@@ -45,7 +45,7 @@ Dynamic: license-file
 # loaderx
 A record-based data runtime, focused on delivering extreme throughput and low latency
 
-**Important Reminder**: Only support Linux_amd64. This is because we use a dynamic library built with Zig (we build and test it on Debian).
+**Only Python3.13_Linux_amd64**
 
 ## Sampler
 a high-performance sampler implemented in Zig

{loaderx-0.2.1 → loaderx-0.3.0}/README.md

@@ -1,7 +1,7 @@
 # loaderx
 A record-based data runtime, focused on delivering extreme throughput and low latency
 
-**Important Reminder**: Only support Linux_amd64. This is because we use a dynamic library built with Zig (we build and test it on Debian).
+**Only Python3.13_Linux_amd64**
 
 ## Sampler
 a high-performance sampler implemented in Zig

{loaderx-0.2.1 → loaderx-0.3.0}/loaderx/__init__.py

@@ -2,4 +2,4 @@ from .dataset import BaseDataset, NPDataset, ARDataset
 from .dataloader import DataLoader
 from .utils import converter
 
-__version__ = "0.2.1"
+__version__ = "0.3.0"

loaderx-0.3.0/loaderx/_sampler.py

@@ -0,0 +1,69 @@
+import os
+import time
+import ctypes
+import numpy as np
+from typing import Union
+
+from lib import libsampler
+
+class Sampler:
+    class Mode:
+        SEQUENTIAL = 0
+        IID = 1
+
+    def __init__(self, length: int, batch_size: int, mode: int, seed: int = 42):
+        # sampler
+        self.sampler = libsampler.Sampler.init(length, batch_size, mode, seed)
+        # indices
+        self.indices = np.zeros(batch_size, dtype=np.uint64)
+
+    # step
+    def next(self):
+        self.sampler.next(self.indices)
+
+    # iterator
+    def __iter__(self):
+        return self
+    def __next__(self):
+        self.next()
+        return self.indices
+
+def run():
+    from itertools import islice
+    length = 10
+    batch_size = 4
+    n_steps = 5
+
+    print("=== Sampler SEQUENTIAL ===")
+    sampler = Sampler(length, batch_size, Sampler.Mode.SEQUENTIAL)
+    for indices in islice(sampler, n_steps):
+        print(indices)
+
+    print("=== Sampler IID ===")
+    sampler = Sampler(length, batch_size, Sampler.Mode.IID)
+    for indices in islice(sampler, n_steps):
+        print(indices)
+
+# Benchmark, zig-sampler speeder 2.2x
+def bench():
+    length = 1_000_000
+    batch_size = 8192
+    n_steps = 10_000
+
+    print("=== NumPy IID ===")
+    t0 = time.time()
+    batch = np.zeros(batch_size, dtype=np.uint64)
+    for _ in range(n_steps):
+        batch[:] = np.random.randint(0, length, size=batch_size, dtype=np.uint64)
+    print(f"{(time.time() - t0)*1000:.2f} ms")
+
+    print("=== Sampler IID ===")
+    t0 = time.time()
+    sampler = Sampler(length, batch_size, Sampler.Mode.IID)
+    for _ in range(n_steps):
+        sampler.next()
+    print(f"{(time.time() - t0)*1000:.2f} ms")
+
+if __name__ == "__main__":
+    run()
+    bench()

{loaderx-0.2.1 → loaderx-0.3.0}/loaderx/dataloader.py

@@ -47,8 +47,8 @@ class DataLoader:
         Sample indices from the dataset and put them into the index queue.
         """
         while not self.stop_signal.is_set():
-            self.sampler.next()   
-            self.indices.put(self.sampler.batch_indices.copy())
+            self.sampler.next()
+            self.indices.put(self.sampler.indices.copy())
 
     def _prefetch(self, transform):
         """

{loaderx-0.2.1 → loaderx-0.3.0}/loaderx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loaderx
-Version: 0.2.1
+Version: 0.3.0
 Summary: A record-based data runtime, focused on delivering extreme throughput and low latency
 Author-email: Ben0i0d <ben0i0d@foxmail.com>
 License: MIT License
@@ -34,7 +34,7 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.10
+Requires-Python: ==3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
@@ -45,7 +45,7 @@ Dynamic: license-file
 # loaderx
 A record-based data runtime, focused on delivering extreme throughput and low latency
 
-**Important Reminder**: Only support Linux_amd64. This is because we use a dynamic library built with Zig (we build and test it on Debian).
+**Only Python3.13_Linux_amd64**
 
 ## Sampler
 a high-performance sampler implemented in Zig

{loaderx-0.2.1 → loaderx-0.3.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = "loaderx"
 dynamic = ["version"]
 description = "A record-based data runtime, focused on delivering extreme throughput and low latency"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = "==3.13"
 authors = [
     {name = "Ben0i0d", email = "ben0i0d@foxmail.com"},
 ]

loaderx-0.2.1/loaderx/_sampler.py

@@ -1,114 +0,0 @@
-import os
-import time
-import ctypes
-import numpy as np
-from typing import Union
-
-class Sampler:
-    
-    class Mode:
-        SEQUENTIAL = 0
-        IID = 1
-    
-    # lazy import
-    _lib = None
-    @classmethod
-    def _load_lib(cls):
-        if cls._lib is None:
-            # linux/windows/macos
-            names = ['lib/libsampler.so', 'lib/sampler.dll', 'lib/libsampler.dylib']
-            lib_paths = [os.path.join(os.path.dirname(__file__), name) for name in names]
-
-            for path in lib_paths:
-                if os.path.exists(path):
-                        cls._lib = ctypes.CDLL(path)
-                        cls._setup_function_signatures()
-        return cls._lib
-    
-    @classmethod
-    def _setup_function_signatures(cls):
-        lib = cls._lib
-        
-        # init
-        lib.init.argtypes = [
-            ctypes.c_void_p,  # sampler
-            ctypes.c_uint64,  # length
-            ctypes.c_uint64,  # batch_size
-            ctypes.c_uint8,   # mode
-            ctypes.c_uint64   # seed
-        ]
-        lib.init.restype = None
-        
-        # size
-        lib.size.restype = ctypes.c_size_t
-
-        # next
-        lib.next.argtypes = [
-            ctypes.c_void_p,     # sampler
-            ctypes.c_void_p      # batch_indices
-        ]
-        lib.next.restype = None
-
-    def __init__(self, length: int, batch_size: int, mode: int, seed: int = 42):
-        # import lib
-        self.lib = self._load_lib()
-
-        # sampler
-        self.sampler = ctypes.create_string_buffer(self.lib.size())
-        self.sampler_ptr = ctypes.cast(self.sampler, ctypes.c_void_p)
-        self.lib.init(self.sampler_ptr, length, batch_size, mode, seed)
-        
-        # batch_indices
-        self.indices = np.zeros(batch_size, dtype=np.uint64)
-        self.indices_ptr = self.indices.ctypes.data
-
-    # step
-    def next(self):
-        self.lib.next(self.sampler_ptr, self.indices_ptr)
-
-    # iterator
-    def __iter__(self):
-        return self
-    def __next__(self):
-        self.next()
-        return self.indices
-
-def run():
-    from itertools import islice
-    length = 10
-    batch_size = 4
-    n_steps = 5
-
-    print("=== Sampler SEQUENTIAL ===")
-    sampler = Sampler(length, batch_size, Sampler.Mode.SEQUENTIAL)
-    for indices in islice(sampler, n_steps):
-        print(indices)
-
-    print("=== Sampler IID ===")
-    sampler = Sampler(length, batch_size, Sampler.Mode.IID)
-    for indices in islice(sampler, n_steps):
-        print(indices)
-
-# Benchmark, zig-sampler speeder 2.2x
-def bench():
-    length = 1_000_000
-    batch_size = 8192
-    n_steps = 10_000
-
-    print("=== NumPy IID ===")
-    t0 = time.time()
-    batch = np.zeros(batch_size, dtype=np.uint64)
-    for _ in range(n_steps):
-        batch[:] = np.random.randint(0, length, size=batch_size, dtype=np.uint64)
-    print(f"{(time.time() - t0)*1000:.2f} ms")
-
-    print("=== Sampler IID ===")
-    t0 = time.time()
-    sampler = Sampler(length, batch_size, Sampler.Mode.IID)
-    for _ in range(n_steps):
-        sampler.next()
-    print(f"{(time.time() - t0)*1000:.2f} ms")
-
-if __name__ == "__main__":
-    run()
-    bench()