dgen-py 0.1.2__cp310-cp310-manylinux_2_24_x86_64.whl

dgen_py/__init__.py

@@ -0,0 +1,167 @@
+"""
+dgen-py: High-performance random data generation with NUMA optimization
+
+TRUE ZERO-COPY: Uses Python buffer protocol for zero-copy access to generated data.
+No memcpy between Rust and Python - same performance as numpy!
+"""
+
+from typing import Optional
+import sys
+
+# Import Rust extension module
+try:
+    from ._dgen_rs import (
+        BytesView,
+        generate_buffer,
+        generate_into_buffer,
+        Generator,
+    )
+    
+    # Try to import NUMA info (may not be available on all platforms)
+    try:
+        from ._dgen_rs import get_numa_info
+    except ImportError:
+        get_numa_info = None
+        
+except ImportError as e:
+    raise ImportError(
+        f"Failed to import dgen-py Rust extension: {e}\n"
+        "Please ensure the package is properly installed:\n"
+        "  pip install dgen-py\n"
+        "Or build from source:\n"
+        "  cd dgen-rs && maturin develop --release"
+    )
+
+__version__ = "0.1.1"
+__all__ = [
+    "BytesView",
+    "generate_buffer",
+    "generate_data",
+    "generate_into_buffer",
+    "fill_buffer",
+    "Generator",
+    "get_numa_info",
+    "get_system_info",
+]
+
+
+def generate_data(
+    size: int,
+    dedup_ratio: float = 1.0,
+    compress_ratio: float = 1.0,
+    numa_mode: str = "auto",
+    max_threads: Optional[int] = None,
+):
+    """
+    Generate random data with ZERO-COPY access via buffer protocol.
+    
+    Returns a BytesView object that supports memoryview() for true zero-copy access.
+    No memcpy between Rust and Python - same memory is shared!
+    
+    Args:
+        size: Total bytes to generate
+        dedup_ratio: Deduplication ratio (1.0 = no dedup, 2.0 = 2:1 ratio)
+        compress_ratio: Compression ratio (1.0 = incompressible, 3.0 = 3:1 ratio)
+        numa_mode: NUMA optimization - \"auto\" (default), \"force\", or \"disabled\"
+        max_threads: Maximum threads to use (None = use all cores)
+    
+    Returns:
+        BytesView: Zero-copy buffer (use memoryview() or numpy.frombuffer() for access)
+    
+    Example - Zero-copy with numpy (fastest):
+        >>> import dgen_py
+        >>> import numpy as np
+        >>> 
+        >>> # Generate data (no copy)
+        >>> data = dgen_py.generate_data(1024 * 1024)
+        >>> 
+        >>> # Create memoryview (no copy)
+        >>> view = memoryview(data)
+        >>> 
+        >>> # Create numpy array (STILL no copy!)
+        >>> arr = np.frombuffer(view, dtype=np.uint8)
+        >>> 
+        >>> # All three share the SAME memory - zero copies!
+        >>> len(data), len(view), len(arr)
+        (1048576, 1048576, 1048576)
+        
+    Example - Get Python bytes (copies data):
+        >>> # If you need actual bytes object, call bytes()
+        >>> data_bytes = bytes(data)  # This copies, but gives you bytes object
+    """
+    return generate_buffer(size, dedup_ratio, compress_ratio, numa_mode, max_threads)
+
+
+def fill_buffer(
+    buffer,
+    dedup_ratio: float = 1.0,
+    compress_ratio: float = 1.0,
+    numa_mode: str = "auto",
+    max_threads: Optional[int] = None,
+) -> int:
+    """
+    Generate data directly into an existing buffer (zero-copy).
+    
+    This is the most efficient API for pre-allocated buffers.
+    Works with bytearray, memoryview, numpy arrays, etc.
+    
+    Args:
+        buffer: Pre-allocated writable buffer (supports buffer protocol)
+        dedup_ratio: Deduplication ratio
+        compress_ratio: Compression ratio
+        numa_mode: NUMA optimization - "auto" (default), "force", or "disabled"
+        max_threads: Maximum threads to use (None = use all cores)
+    
+    Returns:
+        int: Number of bytes written
+    
+    Example:
+        >>> import dgen_py
+        >>> 
+        >>> # Pre-allocate buffer
+        >>> buf = bytearray(1024 * 1024)
+        >>> 
+        >>> # Generate directly into buffer (zero-copy) using 4 threads
+        >>> nbytes = dgen_py.fill_buffer(buf, compress_ratio=2.0, max_threads=4)
+        >>> print(f"Wrote {nbytes} bytes")
+        
+        >>> # Works with numpy arrays
+        >>> import numpy as np
+        >>> arr = np.zeros(1024 * 1024, dtype=np.uint8)
+        >>> nbytes = dgen_py.fill_buffer(arr, dedup_ratio=2.0)
+    """
+    return generate_into_buffer(buffer, dedup_ratio, compress_ratio, numa_mode, max_threads)
+
+
+def get_system_info() -> Optional[dict]:
+    """
+    Get NUMA topology information (if available).
+    
+    Returns:
+        dict: NUMA info with keys:
+            - num_nodes: Number of NUMA nodes
+            - physical_cores: Total physical cores
+            - logical_cpus: Total logical CPUs
+            - is_uma: Whether this is a UMA system
+            - deployment_type: Description of deployment type
+        None: If NUMA detection is not available on this platform
+    
+    Example:
+        >>> info = dgen_py.get_system_info()
+        >>> if info:
+        ...     print(f"NUMA nodes: {info['num_nodes']}")
+        ...     print(f"Cores: {info['physical_cores']}")
+        ...     print(f"Type: {info['deployment_type']}")
+    """
+    if get_numa_info is None:
+        return None
+    
+    try:
+        return get_numa_info()
+    except Exception:
+        return None
+
+
+# Convenience alias
+StreamingGenerator = Generator
+

dgen_py/__init__.pyi

@@ -0,0 +1,61 @@
+"""Type stubs for dgen-py"""
+
+from typing import Optional
+
+def generate_buffer(
+    size: int,
+    dedup_ratio: float = 1.0,
+    compress_ratio: float = 1.0
+) -> bytes:
+    """Generate random data with controllable characteristics"""
+    ...
+
+def generate_into_buffer(
+    buffer,
+    dedup_ratio: float = 1.0,
+    compress_ratio: float = 1.0
+) -> int:
+    """Generate data directly into existing buffer (zero-copy)"""
+    ...
+
+class Generator:
+    """Streaming data generator"""
+    
+    def __init__(
+        self,
+        size: int,
+        dedup_ratio: float = 1.0,
+        compress_ratio: float = 1.0,
+        numa_mode: str = "auto",
+        max_threads: Optional[int] = None
+    ) -> None:
+        """Create new generator"""
+        ...
+    
+    def fill_chunk(self, buffer) -> int:
+        """Fill next chunk into buffer"""
+        ...
+    
+    def get_chunk(self, chunk_size: int) -> Optional[bytes]:
+        """Get next chunk as bytes"""
+        ...
+    
+    def reset(self) -> None:
+        """Reset to start"""
+        ...
+    
+    def position(self) -> int:
+        """Get current position"""
+        ...
+    
+    def total_size(self) -> int:
+        """Get total size"""
+        ...
+    
+    def is_complete(self) -> bool:
+        """Check if complete"""
+        ...
+
+def get_numa_info() -> dict:
+    """Get NUMA topology information"""
+    ...

dgen_py/docs/PERFORMANCE.md

@@ -0,0 +1,241 @@
+# Performance Benchmarks
+
+## dgen-py vs Numpy Random Generation
+
+**Test System**: 12-core UMA system (single NUMA node)  
+**Date**: January 8, 2026  
+**Python**: 3.12  
+**Numpy**: Latest  
+**Test Method**: 5 runs per size, averaged
+
+---
+
+## Benchmark Results
+
+### Performance Comparison Table
+
+| Size    | Method                  | Time (ms) | Throughput | vs dgen-py |
+|---------|-------------------------|-----------|------------|------------|
+| **1 MiB**   | dgen-py             | 2.7 ms    | 0.39 GB/s  | baseline   |
+|         | numpy.random.randint    | 11.8 ms   | 0.09 GB/s  | **4.36x**  |
+|         | numpy.random.bytes      | 1.1 ms    | 0.98 GB/s  | 0.39x      |
+| **10 MiB**  | dgen-py             | 7.1 ms    | 1.48 GB/s  | baseline   |
+|         | numpy.random.randint    | 15.3 ms   | 0.69 GB/s  | **2.15x**  |
+|         | numpy.random.bytes      | 11.4 ms   | 0.92 GB/s  | **1.60x**  |
+| **100 MiB** | dgen-py             | 14.4 ms   | 7.26 GB/s  | baseline   |
+|         | numpy.random.randint    | 152.6 ms  | 0.69 GB/s  | **10.56x** |
+|         | numpy.random.bytes      | 219.0 ms  | 0.48 GB/s  | **15.16x** |
+| **500 MiB** | dgen-py             | 59.2 ms   | 8.86 GB/s  | baseline   |
+|         | numpy.random.randint    | 763.8 ms  | 0.69 GB/s  | **12.91x** |
+|         | numpy.random.bytes      | 1097.8 ms | 0.48 GB/s  | **18.56x** |
+
+---
+
+## Key Findings
+
+### 🚀 Performance Summary
+
+- **Average speedup vs numpy.random.randint**: **7.50x faster**
+- **Average speedup vs numpy.random.bytes**: **8.93x faster**
+- **Peak throughput**: **8.86 GB/s** (at 500 MiB)
+- **Best speedup**: **18.56x faster** than numpy.random.bytes at 500 MiB
+
+### 📊 Throughput Scaling
+
+```
+Size      dgen-py    numpy.randint  numpy.bytes   Speedup (best)
+--------------------------------------------------------------
+1 MiB     0.39 GB/s  0.09 GB/s      0.98 GB/s     0.39x (slower)
+10 MiB    1.48 GB/s  0.69 GB/s      0.92 GB/s     1.60x
+100 MiB   7.26 GB/s  0.69 GB/s      0.48 GB/s     15.16x
+500 MiB   8.86 GB/s  0.69 GB/s      0.48 GB/s     18.56x
+```
+
+**Observation**: dgen-py scales linearly with data size due to multi-threading, while numpy performance plateaus (single-threaded).
+
+---
+
+## Why dgen-py is Faster
+
+### 1. **Superior RNG Algorithm**
+- **dgen-py**: Xoshiro256++ (fastest high-quality RNG)
+- **numpy**: MT19937 (Mersenne Twister, slower but proven)
+- Xoshiro256++ provides ~2x raw speed advantage
+
+### 2. **Multi-Threading**
+- **dgen-py**: Rayon-based parallel generation across all cores
+- **numpy**: Single-threaded random number generation
+- Linear scaling on multi-core systems (12 cores = ~12x potential)
+
+### 3. **Zero-Copy Architecture**
+- **dgen-py**: Buffer protocol (`__getbuffer__`) for direct memory access
+- **numpy**: Must allocate numpy array, copy data
+- Eliminates allocation overhead and memcpy latency
+
+### 4. **Optimized for Bulk Generation**
+- Pre-allocated buffers
+- Cache-friendly memory access patterns
+- First-touch NUMA locality (on NUMA systems)
+
+---
+
+## When to Use Each
+
+### Use dgen-py when:
+✅ Generating **large datasets** (100+ MiB)  
+✅ Need **maximum throughput** (AI/ML data generation)  
+✅ Working with **binary data** (files, network buffers)  
+✅ Have **multi-core system** to leverage parallelism  
+✅ Need **zero-copy integration** with other tools
+
+### Use numpy.random when:
+✅ Generating **small arrays** (<10 MiB)  
+✅ Need **statistical distributions** (normal, poisson, etc.)  
+✅ Need **specific random seeds** for reproducibility  
+✅ Integration with **numpy-centric workflow**  
+✅ Need **element-wise operations** on random data
+
+---
+
+## Zero-Copy Verification
+
+### Memory Access Pattern
+
+```python
+import dgen_py
+import numpy as np
+
+# Generate data (Rust allocation)
+data = dgen_py.generate_data(100 * 1024 * 1024)  # 100 MiB
+
+# Create memoryview (zero-copy, <2 µs)
+view = memoryview(data)
+
+# Create numpy array (zero-copy, <10 µs)
+arr = np.frombuffer(view, dtype=np.uint8)
+
+# All three share THE SAME memory:
+assert len(data) == len(view) == len(arr)  # 104,857,600 bytes
+```
+
+**Memory Overhead**:
+- **With copy**: 300 MiB (3 allocations)
+- **Zero-copy**: 100 MiB (1 allocation)
+- **Savings**: 66% memory reduction
+
+**Performance Overhead**:
+- Memoryview creation: **~1 µs**
+- Numpy array creation: **~8 µs**
+- **Total**: <10 µs (negligible compared to generation time)
+
+---
+
+## Comparison to Other Tools
+
+### Throughput Comparison (500 MiB test)
+
+| Tool                    | Throughput | Notes                           |
+|-------------------------|------------|---------------------------------|
+| **dgen-py**             | 8.86 GB/s  | Multi-threaded, zero-copy       |
+| numpy.random.bytes      | 0.48 GB/s  | Single-threaded                 |
+| numpy.random.randint    | 0.69 GB/s  | Single-threaded + array overhead|
+| dd if=/dev/urandom      | ~0.05 GB/s | Kernel RNG (cryptographic)      |
+| Rust (native)           | 12.5 GB/s  | Direct benchmark, no Python overhead |
+
+**Note**: dgen-py achieves **70% of native Rust performance** through Python, demonstrating the effectiveness of zero-copy design.
+
+---
+
+## Technical Details
+
+### Test Configuration
+
+```python
+# Benchmark parameters
+sizes = [1 MiB, 10 MiB, 100 MiB, 500 MiB]
+runs_per_size = 5
+numa_mode = "auto"  # Auto-detect NUMA topology
+max_threads = None  # Use all available cores (12)
+```
+
+### System Configuration
+
+- **CPU**: 12 cores (UMA, single NUMA node)
+- **Memory**: Standard system allocator
+- **Python**: 3.12 with uv virtual environment
+- **Compiler**: rustc 1.91+ with LTO enabled
+- **Build**: `maturin develop --release`
+
+### Benchmark Script
+
+Run the benchmark yourself:
+
+```bash
+cd dgen-rs
+python python/examples/benchmark_vs_numpy.py
+```
+
+---
+
+## Recommendations for Production
+
+### For AI/ML Training Data Generation
+
+```python
+import dgen_py
+import numpy as np
+
+# Generate 1 GB of random data
+data = dgen_py.generate_data(1024 * 1024 * 1024, numa_mode="auto")
+
+# Zero-copy conversion to numpy for processing
+view = memoryview(data)
+arr = np.frombuffer(view, dtype=np.float32)  # Reinterpret as float32
+
+# Reshape for model input (e.g., 256x256 RGB images)
+images = arr.reshape(-1, 256, 256, 3)
+```
+
+**Expected Performance**: ~8-10 GB/s on typical workstation
+
+### For Storage Benchmarking
+
+```python
+import dgen_py
+
+# Generate incompressible data for realistic I/O testing
+data = dgen_py.generate_data(
+    size=10 * 1024**3,        # 10 GB
+    compress_ratio=1.0,        # Incompressible
+    dedup_ratio=1.0,           # No deduplication
+    numa_mode="force"          # Force NUMA optimizations
+)
+
+# Write to storage (data is bytes-like, works with file I/O)
+with open('/mnt/storage/testfile.bin', 'wb') as f:
+    f.write(bytes(data))  # Converts from BytesView to bytes
+```
+
+---
+
+## Future Optimizations
+
+Potential improvements for even better performance:
+
+1. **SIMD Instructions**: AVX-512 for 2-4x speedup on modern CPUs
+2. **GPU Generation**: CUDA/ROCm for 100+ GB/s on high-end GPUs
+3. **Async I/O Integration**: Direct-to-disk generation without intermediate buffers
+4. **Custom Allocators**: jemalloc/mimalloc for better multi-threaded allocation
+
+---
+
+## Conclusion
+
+dgen-py delivers **production-grade performance** for random data generation:
+
+- ✅ **7-18x faster** than numpy for bulk generation
+- ✅ **True zero-copy** via Python buffer protocol
+- ✅ **Multi-threaded** scaling on modern CPUs
+- ✅ **Competitive with native Rust** (70% of raw performance)
+
+For AI/ML workloads requiring large amounts of random data, dgen-py provides a **significant performance advantage** over traditional numpy-based approaches.

dgen_py/examples/README.md

@@ -0,0 +1,201 @@
+# Python Examples
+
+Performance benchmark scripts to find optimal dgen-py settings for your system.
+
+## Quick Start
+
+```bash
+# Create virtual environment with uv
+uv venv --python 3.12
+source .venv/bin/activate  # or: .venv\Scripts\activate on Windows
+
+# Build and install dgen-py
+maturin develop --release
+
+# Run quick 30-second test
+python python/examples/quick_perf_test.py
+```
+
+## Scripts
+
+### 1. quick_perf_test.py - Fast Optimization (30 seconds)
+
+Quick test to find the best settings for your system.
+
+```bash
+python python/examples/quick_perf_test.py
+```
+
+**Tests:**
+- Default (auto-detect)
+- Force NUMA mode
+- NUMA disabled
+- Half thread count
+- Single thread baseline
+
+**Output:** Ranked results with recommended configuration
+
+---
+
+### 2. benchmark_cpu_numa.py - Comprehensive Benchmark (5-10 minutes)
+
+Deep performance analysis with 4 benchmark suites.
+
+```bash
+python python/examples/benchmark_cpu_numa.py
+```
+
+**Benchmark Suites:**
+
+1. **Thread Scaling** - Test 1,2,4,8,16 threads and all cores
+2. **NUMA Modes** - Compare auto/force/disabled
+3. **Compression Impact** - Test 1x, 2x, 3x, 5x compression ratios
+4. **Optimal Config** - Find best thread count + NUMA mode combination
+
+**Output:**
+- Per-suite performance charts (text-based)
+- Detailed throughput tables
+- Optimal configuration recommendations
+- CSV results in `benchmark_results/`
+
+---
+
+## Example Output
+
+### quick_perf_test.py
+```
+dgen-py Quick Performance Test
+==================================================
+
+System: 1 NUMA node(s), 12 CPUs
+  → Single-socket system (UMA)
+
+Running tests...
+--------------------------------------------------
+
+1. Default (auto-detect)... 1.05 GB/s
+2. Force NUMA... 1.04 GB/s
+3. NUMA disabled... 1.08 GB/s
+4. Half threads (6)... 1.12 GB/s
+5. Single thread (baseline)... 0.73 GB/s
+
+==================================================
+RESULTS (fastest to slowest):
+==================================================
+★ 1. Half threads (6)              1.12 GB/s
+  2. NUMA disabled                 1.08 GB/s
+  3. Default (auto)                1.05 GB/s
+
+==================================================
+RECOMMENDATION: Half threads (6)
+  Throughput: 1.12 GB/s
+  Code: dgen_py.generate_data(size, max_threads=6)
+==================================================
+```
+
+---
+
+## System Requirements
+
+- **Python**: 3.8+ (3.12 recommended via uv)
+- **dgen-py**: Built from source with `maturin develop --release`
+- **OS**: Linux (best performance), macOS, Windows
+
+## Performance Tips
+
+### UMA Systems (Cloud VMs, Workstations)
+- Use `numa_mode="disabled"` to avoid detection overhead
+- Experiment with thread counts (often half or 3/4 of cores is optimal)
+- Single-socket systems won't benefit from NUMA optimizations
+
+### NUMA Systems (Bare Metal, Multi-Socket)
+- Use `numa_mode="auto"` (default) for intelligent detection
+- Use `numa_mode="force"` to force optimizations
+- Expect **30-50% throughput improvement** from thread pinning + first-touch
+- Run benchmarks on actual hardware to measure gains
+
+### General
+- Always test on your target hardware
+- Compression ratio affects optimal thread count
+- I/O-bound workloads may benefit from more threads
+- CPU-bound workloads may saturate with fewer threads
+
+---
+
+## Interpreting Results
+
+### Throughput (GB/s)
+- **< 1 GB/s**: Single thread or small dataset
+- **1-5 GB/s**: Good multi-threaded UMA performance
+- **5-10 GB/s**: Excellent UMA or good NUMA performance
+- **10-20 GB/s**: Excellent NUMA with optimizations
+
+### Scaling Efficiency
+```python
+efficiency = (throughput_N_threads / throughput_1_thread) / N
+```
+- **> 0.8**: Excellent scaling (near-linear)
+- **0.5-0.8**: Good scaling
+- **< 0.5**: Poor scaling (reduce thread count)
+
+### NUMA vs UMA
+- **UMA systems**: Force/Auto should show similar performance
+- **NUMA systems**: Force should outperform Disabled by 30-50%
+
+---
+
+## Development
+
+To modify these scripts:
+
+```bash
+# Edit the scripts
+vim python/examples/quick_perf_test.py
+
+# Rebuild if you changed Rust code
+maturin develop --release
+
+# Re-run tests
+python python/examples/quick_perf_test.py
+```
+
+---
+
+## Troubleshooting
+
+### ModuleNotFoundError: No module named 'dgen_py'
+
+Run `maturin develop --release` to build and install the package.
+
+### ImportError: cannot import name 'generate_data'
+
+Your dgen-py installation is outdated. Rebuild:
+```bash
+maturin develop --release --force
+```
+
+### Low Performance on NUMA Systems
+
+1. Verify NUMA detection: `python -c "import dgen_py; print(dgen_py.get_system_info())"`
+2. Check `num_nodes` > 1
+3. Try `numa_mode="force"`
+4. Ensure running on actual NUMA hardware (not VM)
+
+### Performance Varies Between Runs
+
+- Normal variation: ±5%
+- Large variation: System is busy, close background apps
+- Benchmark script runs warmup iterations to stabilize
+
+---
+
+## Contributing
+
+Found optimal settings for your hardware? Share them!
+
+Create an issue or PR with:
+- Hardware specs (CPU, sockets, NUMA topology)
+- Benchmark results
+- Optimal configuration
+
+This helps us improve auto-detection and recommendations.