turboloader 2.3.2__tar.gz → 2.3.4__tar.gz

{turboloader-2.3.2 → turboloader-2.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: turboloader
-Version: 2.3.2
+Version: 2.3.4
 Summary: Production-ready ML data loading library with distributed training support, SIMD-accelerated transforms, pipe operator composition, HDF5/TFRecord/Zarr support, and GPU transforms. Built with C++20 for maximum performance.
 Author: TurboLoader Contributors
 Author-email: Arnav Jain <arnav@example.com>

{turboloader-2.3.2 → turboloader-2.3.4}/benchmarks/datasets/generate_synthetic.py

@@ -22,6 +22,7 @@ import numpy as np
 
 try:
     from PIL import Image
+
     HAS_PIL = True
 except ImportError:
     HAS_PIL = False
@@ -58,19 +59,19 @@ def generate_random_image(width: int, height: int, channels: int = 3) -> np.ndar
 def save_image_jpeg(img: np.ndarray, path: str, quality: int = 85):
     """Save image as JPEG"""
     if HAS_PIL:
-        Image.fromarray(img).save(path, 'JPEG', quality=quality)
+        Image.fromarray(img).save(path, "JPEG", quality=quality)
     else:
         # Fallback: save as raw binary
-        with open(path, 'wb') as f:
+        with open(path, "wb") as f:
             f.write(img.tobytes())
 
 
 def save_image_png(img: np.ndarray, path: str):
     """Save image as PNG"""
     if HAS_PIL:
-        Image.fromarray(img).save(path, 'PNG')
+        Image.fromarray(img).save(path, "PNG")
     else:
-        with open(path, 'wb') as f:
+        with open(path, "wb") as f:
             f.write(img.tobytes())
 
 
@@ -79,8 +80,8 @@ def generate_dataset_files(
     num_images: int,
     image_size: tuple = (256, 256),
     num_classes: int = 1000,
-    format: str = 'jpeg',
-    num_workers: int = 4
+    format: str = "jpeg",
+    num_workers: int = 4,
 ) -> list:
     """Generate dataset as individual image files"""
     os.makedirs(output_dir, exist_ok=True)
@@ -92,7 +93,7 @@ def generate_dataset_files(
         img = generate_random_image(image_size[0], image_size[1])
         label = idx % num_classes
 
-        if format == 'jpeg':
+        if format == "jpeg":
             filename = f"img_{idx:08d}.jpg"
             filepath = os.path.join(output_dir, filename)
             save_image_jpeg(img, filepath)
@@ -116,8 +117,8 @@ def generate_dataset_files(
                 print(f"  Generated {i + 1}/{num_images} images")
 
     # Save labels file
-    labels_path = os.path.join(output_dir, 'labels.txt')
-    with open(labels_path, 'w') as f:
+    labels_path = os.path.join(output_dir, "labels.txt")
+    with open(labels_path, "w") as f:
         for filepath, label in zip(files, labels):
             f.write(f"{os.path.basename(filepath)},{label}\n")
 
@@ -130,14 +131,14 @@ def generate_tar_dataset(
     num_images: int,
     image_size: tuple = (256, 256),
     num_classes: int = 1000,
-    format: str = 'jpeg'
+    format: str = "jpeg",
 ) -> str:
     """Generate dataset as a TAR archive"""
     import io
 
     print(f"Generating TAR archive with {num_images} images...")
 
-    with tarfile.open(output_path, 'w') as tar:
+    with tarfile.open(output_path, "w") as tar:
         labels = []
 
         for idx in range(num_images):
@@ -147,11 +148,11 @@ def generate_tar_dataset(
             # Create in-memory file
             if HAS_PIL:
                 buf = io.BytesIO()
-                if format == 'jpeg':
-                    Image.fromarray(img).save(buf, 'JPEG', quality=85)
+                if format == "jpeg":
+                    Image.fromarray(img).save(buf, "JPEG", quality=85)
                     filename = f"img_{idx:08d}.jpg"
                 else:
-                    Image.fromarray(img).save(buf, 'PNG')
+                    Image.fromarray(img).save(buf, "PNG")
                     filename = f"img_{idx:08d}.png"
                 buf.seek(0)
                 data = buf.getvalue()
@@ -170,9 +171,9 @@ def generate_tar_dataset(
                 print(f"  Added {idx + 1}/{num_images} images to TAR")
 
         # Add labels file
-        labels_content = '\n'.join(f"{f},{l}" for f, l in labels)
-        labels_data = labels_content.encode('utf-8')
-        info = tarfile.TarInfo(name='labels.txt')
+        labels_content = "\n".join(f"{f},{l}" for f, l in labels)
+        labels_data = labels_content.encode("utf-8")
+        info = tarfile.TarInfo(name="labels.txt")
         info.size = len(labels_data)
         tar.addfile(info, io.BytesIO(labels_data))
 
@@ -185,7 +186,7 @@ def generate_varying_size_dataset(
     num_images: int,
     min_size: int = 128,
     max_size: int = 512,
-    num_classes: int = 1000
+    num_classes: int = 1000,
 ) -> str:
     """Generate dataset with varying image sizes (for smart batching benchmarks)"""
     import io
@@ -193,7 +194,7 @@ def generate_varying_size_dataset(
     print(f"Generating varying-size TAR archive with {num_images} images...")
 
     sizes = []
-    with tarfile.open(output_path, 'w') as tar:
+    with tarfile.open(output_path, "w") as tar:
         labels = []
 
         for idx in range(num_images):
@@ -207,7 +208,7 @@ def generate_varying_size_dataset(
 
             if HAS_PIL:
                 buf = io.BytesIO()
-                Image.fromarray(img).save(buf, 'JPEG', quality=85)
+                Image.fromarray(img).save(buf, "JPEG", quality=85)
                 filename = f"img_{idx:08d}.jpg"
                 buf.seek(0)
                 data = buf.getvalue()
@@ -226,12 +227,12 @@ def generate_varying_size_dataset(
 
         # Add metadata
         meta = {
-            'num_images': num_images,
-            'sizes': sizes,
-            'labels': [(f, l) for f, l, w, h in labels]
+            "num_images": num_images,
+            "sizes": sizes,
+            "labels": [(f, l) for f, l, w, h in labels],
         }
-        meta_data = json.dumps(meta).encode('utf-8')
-        info = tarfile.TarInfo(name='metadata.json')
+        meta_data = json.dumps(meta).encode("utf-8")
+        info = tarfile.TarInfo(name="metadata.json")
         info.size = len(meta_data)
         tar.addfile(info, io.BytesIO(meta_data))
 
@@ -240,52 +241,51 @@ def generate_varying_size_dataset(
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Generate synthetic datasets for benchmarking')
-    parser.add_argument('--output', '-o', type=str, required=True,
-                        help='Output path (directory for files, .tar for archive)')
-    parser.add_argument('--num-images', '-n', type=int, default=10000,
-                        help='Number of images to generate')
-    parser.add_argument('--width', type=int, default=256,
-                        help='Image width')
-    parser.add_argument('--height', type=int, default=256,
-                        help='Image height')
-    parser.add_argument('--num-classes', type=int, default=1000,
-                        help='Number of classes for labels')
-    parser.add_argument('--format', choices=['jpeg', 'png'], default='jpeg',
-                        help='Image format')
-    parser.add_argument('--type', choices=['files', 'tar', 'varying'], default='tar',
-                        help='Dataset type (files, tar archive, or varying sizes)')
-    parser.add_argument('--workers', type=int, default=4,
-                        help='Number of worker threads')
+    parser = argparse.ArgumentParser(description="Generate synthetic datasets for benchmarking")
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=str,
+        required=True,
+        help="Output path (directory for files, .tar for archive)",
+    )
+    parser.add_argument(
+        "--num-images", "-n", type=int, default=10000, help="Number of images to generate"
+    )
+    parser.add_argument("--width", type=int, default=256, help="Image width")
+    parser.add_argument("--height", type=int, default=256, help="Image height")
+    parser.add_argument(
+        "--num-classes", type=int, default=1000, help="Number of classes for labels"
+    )
+    parser.add_argument("--format", choices=["jpeg", "png"], default="jpeg", help="Image format")
+    parser.add_argument(
+        "--type",
+        choices=["files", "tar", "varying"],
+        default="tar",
+        help="Dataset type (files, tar archive, or varying sizes)",
+    )
+    parser.add_argument("--workers", type=int, default=4, help="Number of worker threads")
 
     args = parser.parse_args()
 
-    if args.type == 'files':
+    if args.type == "files":
         generate_dataset_files(
             args.output,
             args.num_images,
             (args.width, args.height),
             args.num_classes,
             args.format,
-            args.workers
+            args.workers,
         )
-    elif args.type == 'tar':
+    elif args.type == "tar":
         generate_tar_dataset(
-            args.output,
-            args.num_images,
-            (args.width, args.height),
-            args.num_classes,
-            args.format
+            args.output, args.num_images, (args.width, args.height), args.num_classes, args.format
         )
-    elif args.type == 'varying':
+    elif args.type == "varying":
         generate_varying_size_dataset(
-            args.output,
-            args.num_images,
-            min_size=128,
-            max_size=512,
-            num_classes=args.num_classes
+            args.output, args.num_images, min_size=128, max_size=512, num_classes=args.num_classes
         )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

{turboloader-2.3.2 → turboloader-2.3.4}/benchmarks/memory/bench_memory.py

@@ -23,6 +23,7 @@ import numpy as np
 
 try:
     import psutil
+
     HAS_PSUTIL = True
 except ImportError:
     HAS_PSUTIL = False
@@ -31,12 +32,14 @@ except ImportError:
 try:
     import torch
     from torch.utils.data import DataLoader, Dataset
+
     HAS_TORCH = True
 except ImportError:
     HAS_TORCH = False
 
 try:
     import turboloader
+
     HAS_TURBOLOADER = True
 except ImportError:
     HAS_TURBOLOADER = False
@@ -45,6 +48,7 @@ except ImportError:
 @dataclass
 class MemoryResult:
     """Memory benchmark result"""
+
     library: str
     config: Dict[str, Any]
     baseline_mb: float
@@ -96,18 +100,18 @@ class MemoryMonitor:
         """Get memory statistics"""
         if not self.measurements:
             return {
-                'min_mb': 0,
-                'max_mb': 0,
-                'avg_mb': 0,
-                'std_mb': 0,
+                "min_mb": 0,
+                "max_mb": 0,
+                "avg_mb": 0,
+                "std_mb": 0,
             }
 
         measurements = np.array(self.measurements)
         return {
-            'min_mb': np.min(measurements),
-            'max_mb': np.max(measurements),
-            'avg_mb': np.mean(measurements),
-            'std_mb': np.std(measurements),
+            "min_mb": np.min(measurements),
+            "max_mb": np.max(measurements),
+            "avg_mb": np.mean(measurements),
+            "std_mb": np.std(measurements),
         }
 
 
@@ -126,10 +130,7 @@ def force_gc():
 
 
 def benchmark_turboloader_memory(
-    tar_path: str,
-    batch_size: int = 64,
-    num_workers: int = 4,
-    num_batches: int = 50
+    tar_path: str, batch_size: int = 64, num_workers: int = 4, num_batches: int = 50
 ) -> Optional[MemoryResult]:
     """Benchmark TurboLoader memory usage"""
     if not HAS_TURBOLOADER:
@@ -142,12 +143,14 @@ def benchmark_turboloader_memory(
     monitor = MemoryMonitor(interval=0.05)
 
     try:
-        transforms = turboloader.Compose([
-            turboloader.Resize(256, 256),
-            turboloader.RandomCrop(224, 224),
-            turboloader.RandomHorizontalFlip(0.5),
-            turboloader.ImageNetNormalize(),
-        ])
+        transforms = turboloader.Compose(
+            [
+                turboloader.Resize(256, 256),
+                turboloader.RandomCrop(224, 224),
+                turboloader.RandomHorizontalFlip(0.5),
+                turboloader.ImageNetNormalize(),
+            ]
+        )
 
         loader = turboloader.DataLoader(
             tar_path,
@@ -176,22 +179,22 @@ def benchmark_turboloader_memory(
         monitor.stop()
         stats = monitor.get_stats()
 
-        peak = stats['max_mb']
+        peak = stats["max_mb"]
         delta = peak - baseline
 
         return MemoryResult(
-            library='turboloader',
+            library="turboloader",
             config={
-                'batch_size': batch_size,
-                'num_workers': num_workers,
-                'num_batches': num_batches,
+                "batch_size": batch_size,
+                "num_workers": num_workers,
+                "num_batches": num_batches,
             },
             baseline_mb=baseline,
             peak_mb=peak,
             delta_mb=delta,
-            avg_mb=stats['avg_mb'],
-            samples_per_mb=total_samples / delta if delta > 0 else float('inf'),
-            timestamp=datetime.now().isoformat()
+            avg_mb=stats["avg_mb"],
+            samples_per_mb=total_samples / delta if delta > 0 else float("inf"),
+            timestamp=datetime.now().isoformat(),
         )
 
     except Exception as e:
@@ -205,7 +208,7 @@ def benchmark_pytorch_memory(
     batch_size: int = 64,
     num_workers: int = 4,
     num_batches: int = 50,
-    cached: bool = True
+    cached: bool = True,
 ) -> Optional[MemoryResult]:
     """Benchmark PyTorch DataLoader memory usage"""
     if not HAS_TORCH:
@@ -223,9 +226,9 @@ def benchmark_pytorch_memory(
             self.samples = []
             self.cache = {} if cache else None
 
-            with tarfile.open(tar_path, 'r') as tar:
+            with tarfile.open(tar_path, "r") as tar:
                 for member in tar.getmembers():
-                    if member.name.endswith(('.jpg', '.jpeg', '.png', '.JPEG', '.JPG')):
+                    if member.name.endswith((".jpg", ".jpeg", ".png", ".JPEG", ".JPG")):
                         self.samples.append(member.name)
                         if cache:
                             f = tar.extractfile(member)
@@ -242,11 +245,11 @@ def benchmark_pytorch_memory(
             if self.cache:
                 data = self.cache[filename]
             else:
-                with tarfile.open(self.tar_path, 'r') as tar:
+                with tarfile.open(self.tar_path, "r") as tar:
                     f = tar.extractfile(tar.getmember(filename))
                     data = f.read()
 
-            img = Image.open(BytesIO(data)).convert('RGB')
+            img = Image.open(BytesIO(data)).convert("RGB")
             if self.transform:
                 img = self.transform(img)
 
@@ -259,13 +262,15 @@ def benchmark_pytorch_memory(
     monitor = MemoryMonitor(interval=0.05)
 
     try:
-        transform = T.Compose([
-            T.Resize((256, 256)),
-            T.RandomCrop(224),
-            T.RandomHorizontalFlip(),
-            T.ToTensor(),
-            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
-        ])
+        transform = T.Compose(
+            [
+                T.Resize((256, 256)),
+                T.RandomCrop(224),
+                T.RandomHorizontalFlip(),
+                T.ToTensor(),
+                T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+            ]
+        )
 
         dataset = TarDataset(tar_path, transform=transform, cache=cached)
 
@@ -288,28 +293,29 @@ def benchmark_pytorch_memory(
         monitor.stop()
         stats = monitor.get_stats()
 
-        peak = stats['max_mb']
+        peak = stats["max_mb"]
         delta = peak - baseline
 
         return MemoryResult(
-            library='pytorch' + ('_cached' if cached else ''),
+            library="pytorch" + ("_cached" if cached else ""),
             config={
-                'batch_size': batch_size,
-                'num_workers': num_workers,
-                'num_batches': num_batches,
-                'cached': cached,
+                "batch_size": batch_size,
+                "num_workers": num_workers,
+                "num_batches": num_batches,
+                "cached": cached,
             },
             baseline_mb=baseline,
             peak_mb=peak,
             delta_mb=delta,
-            avg_mb=stats['avg_mb'],
-            samples_per_mb=total_samples / delta if delta > 0 else float('inf'),
-            timestamp=datetime.now().isoformat()
+            avg_mb=stats["avg_mb"],
+            samples_per_mb=total_samples / delta if delta > 0 else float("inf"),
+            timestamp=datetime.now().isoformat(),
         )
 
     except Exception as e:
         print(f"PyTorch error: {e}")
         import traceback
+
         traceback.print_exc()
         monitor.stop()
         return None
@@ -319,7 +325,7 @@ def run_memory_benchmarks(
     tar_path: str,
     batch_sizes: List[int] = [32, 64, 128],
     num_workers: int = 4,
-    num_batches: int = 50
+    num_batches: int = 50,
 ) -> List[MemoryResult]:
     """Run all memory benchmarks"""
     results = []
@@ -337,9 +343,7 @@ def run_memory_benchmarks(
         # TurboLoader
         if HAS_TURBOLOADER:
             print("  TurboLoader...")
-            result = benchmark_turboloader_memory(
-                tar_path, batch_size, num_workers, num_batches
-            )
+            result = benchmark_turboloader_memory(tar_path, batch_size, num_workers, num_batches)
             if result:
                 print(f"    Peak: {result.peak_mb:.1f} MB, Delta: {result.delta_mb:.1f} MB")
                 results.append(result)
@@ -372,10 +376,10 @@ def run_memory_benchmarks(
 
 def save_results(results: List[MemoryResult], output_path: str):
     """Save results to JSON"""
-    os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
+    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
 
     data = [asdict(r) for r in results]
-    with open(output_path, 'w') as f:
+    with open(output_path, "w") as f:
         json.dump(data, f, indent=2)
 
     print(f"\nResults saved to: {output_path}")
@@ -390,23 +394,28 @@ def print_summary(results: List[MemoryResult]):
     print("-" * 80)
 
     for r in results:
-        batch = r.config.get('batch_size', 'N/A')
-        print(f"{r.library:>20} {batch:>8} {r.peak_mb:>12.1f} {r.delta_mb:>12.1f} {r.samples_per_mb:>12.1f}")
+        batch = r.config.get("batch_size", "N/A")
+        print(
+            f"{r.library:>20} {batch:>8} {r.peak_mb:>12.1f} {r.delta_mb:>12.1f} {r.samples_per_mb:>12.1f}"
+        )
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Memory Usage Benchmark')
-    parser.add_argument('--tar-path', type=str, required=True,
-                        help='Path to TAR dataset')
-    parser.add_argument('--batch-sizes', type=int, nargs='+', default=[32, 64, 128],
-                        help='Batch sizes to test')
-    parser.add_argument('--workers', type=int, default=4,
-                        help='Number of workers')
-    parser.add_argument('--num-batches', type=int, default=50,
-                        help='Number of batches per benchmark')
-    parser.add_argument('--output', type=str,
-                        default='benchmarks/results/memory/memory.json',
-                        help='Output path for results')
+    parser = argparse.ArgumentParser(description="Memory Usage Benchmark")
+    parser.add_argument("--tar-path", type=str, required=True, help="Path to TAR dataset")
+    parser.add_argument(
+        "--batch-sizes", type=int, nargs="+", default=[32, 64, 128], help="Batch sizes to test"
+    )
+    parser.add_argument("--workers", type=int, default=4, help="Number of workers")
+    parser.add_argument(
+        "--num-batches", type=int, default=50, help="Number of batches per benchmark"
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="benchmarks/results/memory/memory.json",
+        help="Output path for results",
+    )
 
     args = parser.parse_args()
 
@@ -419,12 +428,12 @@ def main():
         args.tar_path,
         batch_sizes=args.batch_sizes,
         num_workers=args.workers,
-        num_batches=args.num_batches
+        num_batches=args.num_batches,
     )
 
     save_results(results, args.output)
     print_summary(results)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()