dgen-py 0.1.2__cp310-cp310-manylinux_2_24_x86_64.whl

dgen_py/examples/benchmark_cpu_numa.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+"""
+Performance Benchmark: CPU and NUMA Configuration
+==================================================
+
+This script helps you find the optimal CPU and NUMA settings for your system.
+Run this to discover:
+- Optimal thread count for your workload
+- Whether NUMA optimizations help (bare metal) or hurt (cloud VM)
+- Baseline single-core performance
+- Memory bandwidth limits
+
+Usage:
+    python benchmark_cpu_numa.py
+
+Requirements:
+    pip install dgen-py
+"""
+
+import time
+import sys
+import os
+from typing import Optional, List, Tuple
+
+try:
+    import dgen_py
+except ImportError:
+    print("ERROR: dgen-py not installed")
+    print("Install with: pip install dgen-py")
+    print("Or build from source: cd dgen-rs && maturin develop --release")
+    sys.exit(1)
+
+
+class PerformanceBenchmark:
+    """Comprehensive CPU and NUMA performance benchmark"""
+    
+    def __init__(self, size_mb: int = 100):
+        self.size = size_mb * 1024 * 1024
+        self.results = []
+        
+    def run_test(
+        self, 
+        name: str, 
+        dedup_ratio: float = 1.0,
+        compress_ratio: float = 1.0,
+        numa_mode: str = "auto",
+        max_threads: Optional[int] = None,
+        iterations: int = 3
+    ) -> Tuple[float, float]:
+        """Run a single test configuration multiple times and return avg throughput"""
+        
+        times = []
+        for i in range(iterations):
+            start = time.perf_counter()
+            data = dgen_py.generate_data(
+                self.size,
+                dedup_ratio=dedup_ratio,
+                compress_ratio=compress_ratio,
+                numa_mode=numa_mode,
+                max_threads=max_threads
+            )
+            elapsed = time.perf_counter() - start
+            times.append(elapsed)
+            
+            # Verify size
+            if len(data) != self.size:
+                print(f"WARNING: Expected {self.size} bytes, got {len(data)}")
+        
+        avg_time = sum(times) / len(times)
+        throughput = (self.size / avg_time) / 1e9  # GB/s
+        
+        self.results.append({
+            'name': name,
+            'throughput': throughput,
+            'time': avg_time,
+            'threads': max_threads or 'all',
+            'numa_mode': numa_mode,
+            'dedup': dedup_ratio,
+            'compress': compress_ratio
+        })
+        
+        return throughput, avg_time
+    
+    def print_result(self, name: str, throughput: float, time_sec: float, 
+                     threads: Optional[int] = None, numa_mode: str = "auto"):
+        """Print a single test result"""
+        threads_str = f"{threads} threads" if threads else "all cores"
+        print(f"  {name:40s} {throughput:8.2f} GB/s  ({time_sec:.3f}s, {threads_str}, numa={numa_mode})")
+    
+    def benchmark_thread_scaling(self):
+        """Test performance with different thread counts"""
+        print("\n" + "="*80)
+        print("THREAD SCALING BENCHMARK")
+        print("="*80)
+        print("\nTesting different thread counts to find optimal configuration...")
+        print("(All tests: incompressible data, no dedup, NUMA=auto)\n")
+        
+        # Get CPU count for intelligent thread selection
+        import multiprocessing
+        cpu_count = multiprocessing.cpu_count()
+        
+        thread_counts = [1, 2, 4]
+        if cpu_count >= 8:
+            thread_counts.append(8)
+        if cpu_count >= 16:
+            thread_counts.append(16)
+        thread_counts.append(None)  # All cores
+        
+        baseline_throughput = None
+        
+        for threads in thread_counts:
+            name = f"Threads: {threads if threads else cpu_count} ({'baseline' if threads == 1 else 'parallel'})"
+            throughput, elapsed = self.run_test(name, max_threads=threads)
+            self.print_result(name, throughput, elapsed, threads)
+            
+            if threads == 1:
+                baseline_throughput = throughput
+            elif baseline_throughput:
+                speedup = throughput / baseline_throughput
+                efficiency = (speedup / (threads or cpu_count)) * 100
+                print(f"    └─> Speedup: {speedup:.2f}x, Efficiency: {efficiency:.1f}%")
+    
+    def benchmark_numa_modes(self):
+        """Test different NUMA configurations"""
+        print("\n" + "="*80)
+        print("NUMA MODE BENCHMARK")
+        print("="*80)
+        print("\nTesting NUMA optimization modes...")
+        print("(All tests: incompressible data, no dedup, all cores)\n")
+        
+        # Check system NUMA topology
+        numa_info = dgen_py.get_system_info()
+        if numa_info:
+            num_nodes = numa_info['num_nodes']
+            print(f"System: {num_nodes} NUMA node(s) detected")
+            if num_nodes > 1:
+                print(f"  Multi-socket NUMA system - optimizations should help!")
+            else:
+                print(f"  UMA system (single socket) - optimizations add minimal overhead")
+            print()
+        else:
+            print("NUMA detection not available (NUMA feature not compiled in)\n")
+        
+        modes = [
+            ("Auto (default)", "auto"),
+            ("Force NUMA", "force"),
+            ("Disabled", "disabled")
+        ]
+        
+        for name, mode in modes:
+            test_name = f"NUMA mode: {name}"
+            throughput, elapsed = self.run_test(test_name, numa_mode=mode)
+            self.print_result(test_name, throughput, elapsed, numa_mode=mode)
+    
+    def benchmark_compression_impact(self):
+        """Test performance with different compression ratios"""
+        print("\n" + "="*80)
+        print("COMPRESSION IMPACT BENCHMARK")
+        print("="*80)
+        print("\nTesting how compression ratio affects throughput...")
+        print("(All tests: no dedup, NUMA=auto, all cores)\n")
+        
+        compress_ratios = [1, 2, 3, 5]
+        
+        baseline = None
+        for ratio in compress_ratios:
+            name = f"Compression ratio: {ratio}:1 ({'incompressible' if ratio == 1 else 'compressible'})"
+            throughput, elapsed = self.run_test(name, compress_ratio=ratio)
+            self.print_result(name, throughput, elapsed)
+            
+            if ratio == 1:
+                baseline = throughput
+            else:
+                slowdown = baseline / throughput if throughput > 0 else 0
+                print(f"    └─> {slowdown:.2f}x slower than incompressible (more back-refs to copy)")
+    
+    def benchmark_optimal_config(self):
+        """Find optimal configuration for this system"""
+        print("\n" + "="*80)
+        print("OPTIMAL CONFIGURATION FINDER")
+        print("="*80)
+        print("\nTesting combinations to find best performance...\n")
+        
+        import multiprocessing
+        cpu_count = multiprocessing.cpu_count()
+        
+        configs = [
+            ("All cores + Auto NUMA", None, "auto"),
+            ("All cores + Force NUMA", None, "force"),
+            ("All cores + Disabled NUMA", None, "disabled"),
+            ("Half cores + Auto NUMA", cpu_count // 2, "auto"),
+        ]
+        
+        best_throughput = 0
+        best_config = None
+        
+        for name, threads, numa_mode in configs:
+            test_name = f"Config: {name}"
+            throughput, elapsed = self.run_test(test_name, max_threads=threads, numa_mode=numa_mode)
+            self.print_result(test_name, throughput, elapsed, threads, numa_mode)
+            
+            if throughput > best_throughput:
+                best_throughput = throughput
+                best_config = (name, threads, numa_mode)
+        
+        print(f"\n{'='*80}")
+        print(f"WINNER: {best_config[0]}")
+        print(f"  Throughput: {best_throughput:.2f} GB/s")
+        print(f"  Config: max_threads={best_config[1]}, numa_mode='{best_config[2]}'")
+        print(f"{'='*80}")
+    
+    def print_summary(self):
+        """Print overall benchmark summary"""
+        print("\n" + "="*80)
+        print("BENCHMARK SUMMARY")
+        print("="*80)
+        
+        if not self.results:
+            print("No results to summarize")
+            return
+        
+        # Sort by throughput
+        sorted_results = sorted(self.results, key=lambda x: x['throughput'], reverse=True)
+        
+        print(f"\nTop 5 configurations (out of {len(self.results)} tested):\n")
+        print(f"{'Rank':<6} {'Configuration':<40} {'Throughput':<12} {'Settings'}")
+        print("-" * 80)
+        
+        for i, result in enumerate(sorted_results[:5], 1):
+            config = result['name']
+            throughput = f"{result['throughput']:.2f} GB/s"
+            settings = f"threads={result['threads']}, numa={result['numa_mode']}"
+            print(f"{i:<6} {config:<40} {throughput:<12} {settings}")
+        
+        # System recommendations
+        print(f"\n{'='*80}")
+        print("RECOMMENDATIONS FOR YOUR SYSTEM")
+        print(f"{'='*80}\n")
+        
+        best = sorted_results[0]
+        print(f"For maximum performance on this system:")
+        print(f"  dgen_py.generate_data(size,")
+        print(f"      max_threads={repr(best['threads'])},")
+        print(f"      numa_mode='{best['numa_mode']}')")
+        print(f"  Expected: ~{best['throughput']:.1f} GB/s")
+        
+        # Check if NUMA helps
+        numa_info = dgen_py.get_system_info()
+        if numa_info and numa_info['num_nodes'] == 1:
+            print("\nNOTE: This is a UMA system (single NUMA node).")
+            print("  NUMA optimizations have minimal impact here.")
+            print("  On multi-socket bare metal servers, expect 30-50% improvement!")
+
+
+def main():
+    """Run comprehensive benchmark suite"""
+    
+    print("""
+╔══════════════════════════════════════════════════════════════════════════════╗
+║                  dgen-py Performance Benchmark Suite                         ║
+║                  CPU and NUMA Configuration Optimizer                        ║
+╚══════════════════════════════════════════════════════════════════════════════╝
+    """)
+    
+    # Check system info
+    numa_info = dgen_py.get_system_info()
+    if numa_info:
+        print(f"System Information:")
+        print(f"  NUMA nodes: {numa_info['num_nodes']}")
+        print(f"  Physical cores: {numa_info['physical_cores']}")
+        print(f"  Logical CPUs: {numa_info['logical_cpus']}")
+        print(f"  Deployment: {numa_info['deployment_type']}")
+    else:
+        import multiprocessing
+        print(f"System Information:")
+        print(f"  CPUs: {multiprocessing.cpu_count()}")
+        print(f"  NUMA: Not available (feature not compiled in)")
+    
+    print(f"\nBenchmark Configuration:")
+    print(f"  Data size per test: 100 MiB")
+    print(f"  Iterations per config: 3")
+    print(f"  Total time: ~1-2 minutes")
+    
+    # Run benchmarks
+    benchmark = PerformanceBenchmark(size_mb=100)
+    
+    benchmark.benchmark_thread_scaling()
+    benchmark.benchmark_numa_modes()
+    benchmark.benchmark_compression_impact()
+    benchmark.benchmark_optimal_config()
+    benchmark.print_summary()
+    
+    print(f"\n{'='*80}")
+    print("Benchmark complete! Save these results for your production configuration.")
+    print(f"{'='*80}\n")
+
+
+if __name__ == "__main__":
+    main()

dgen_py/examples/benchmark_vs_numpy.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+Benchmark: dgen-py vs Numpy Random
+===================================
+
+Compare dgen-py data generation to numpy's random number generation.
+"""
+
+import dgen_py
+import numpy as np
+import time
+
+
+def benchmark_dgen(size, runs=5):
+    """Benchmark dgen-py generation"""
+    times = []
+    for _ in range(runs):
+        start = time.perf_counter()
+        data = dgen_py.generate_data(size)
+        # Convert to memoryview to ensure zero-copy is used
+        view = memoryview(data)
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+    
+    avg_time = sum(times) / len(times)
+    return avg_time, size / avg_time / 1e9
+
+
+def benchmark_numpy_randint(size, runs=5):
+    """Benchmark numpy random.randint (uint8)"""
+    times = []
+    for _ in range(runs):
+        start = time.perf_counter()
+        arr = np.random.randint(0, 256, size, dtype=np.uint8)
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+    
+    avg_time = sum(times) / len(times)
+    return avg_time, size / avg_time / 1e9
+
+
+def benchmark_numpy_bytes(size, runs=5):
+    """Benchmark numpy random.bytes"""
+    times = []
+    for _ in range(runs):
+        start = time.perf_counter()
+        data = np.random.bytes(size)
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+    
+    avg_time = sum(times) / len(times)
+    return avg_time, size / avg_time / 1e9
+
+
+def main():
+    print("=" * 70)
+    print("BENCHMARK: dgen-py vs Numpy Random Number Generation")
+    print("=" * 70)
+    
+    sizes = [
+        (1 * 1024 * 1024, "1 MiB"),
+        (10 * 1024 * 1024, "10 MiB"),
+        (100 * 1024 * 1024, "100 MiB"),
+        (500 * 1024 * 1024, "500 MiB"),
+    ]
+    
+    print("\nRunning benchmarks (5 runs each, averaged)...\n")
+    
+    results = []
+    
+    for size, label in sizes:
+        print(f"Testing {label}...")
+        
+        # dgen-py
+        dgen_time, dgen_gbps = benchmark_dgen(size)
+        
+        # numpy randint
+        numpy_randint_time, numpy_randint_gbps = benchmark_numpy_randint(size)
+        
+        # numpy bytes
+        numpy_bytes_time, numpy_bytes_gbps = benchmark_numpy_bytes(size)
+        
+        results.append({
+            'size': size,
+            'label': label,
+            'dgen_time': dgen_time,
+            'dgen_gbps': dgen_gbps,
+            'numpy_randint_time': numpy_randint_time,
+            'numpy_randint_gbps': numpy_randint_gbps,
+            'numpy_bytes_time': numpy_bytes_time,
+            'numpy_bytes_gbps': numpy_bytes_gbps,
+        })
+    
+    # Print results table
+    print("\n" + "=" * 70)
+    print("RESULTS")
+    print("=" * 70)
+    print(f"\n{'Size':<10} {'Method':<20} {'Time (ms)':<12} {'Throughput':<15} {'vs dgen':<10}")
+    print("-" * 70)
+    
+    for r in results:
+        # dgen-py
+        print(f"{r['label']:<10} {'dgen-py':<20} {r['dgen_time']*1000:>10.1f} ms {r['dgen_gbps']:>10.2f} GB/s {'baseline':>10}")
+        
+        # numpy randint
+        speedup_randint = r['dgen_gbps'] / r['numpy_randint_gbps']
+        print(f"{'':<10} {'numpy.random.randint':<20} {r['numpy_randint_time']*1000:>10.1f} ms {r['numpy_randint_gbps']:>10.2f} GB/s {speedup_randint:>9.2f}x")
+        
+        # numpy bytes
+        speedup_bytes = r['dgen_gbps'] / r['numpy_bytes_gbps']
+        print(f"{'':<10} {'numpy.random.bytes':<20} {r['numpy_bytes_time']*1000:>10.1f} ms {r['numpy_bytes_gbps']:>10.2f} GB/s {speedup_bytes:>9.2f}x")
+        
+        print()
+    
+    # Summary
+    print("=" * 70)
+    print("SUMMARY")
+    print("=" * 70)
+    
+    # Average speedups
+    avg_speedup_randint = sum(r['dgen_gbps'] / r['numpy_randint_gbps'] for r in results) / len(results)
+    avg_speedup_bytes = sum(r['dgen_gbps'] / r['numpy_bytes_gbps'] for r in results) / len(results)
+    
+    print(f"\ndgen-py average performance:")
+    print(f"  vs numpy.random.randint: {avg_speedup_randint:.2f}x faster")
+    print(f"  vs numpy.random.bytes:   {avg_speedup_bytes:.2f}x faster")
+    
+    # Find best dgen performance
+    best = max(results, key=lambda r: r['dgen_gbps'])
+    print(f"\nPeak dgen-py throughput: {best['dgen_gbps']:.2f} GB/s ({best['label']})")
+    
+    print("\n" + "=" * 70)
+    print("✓ Zero-copy implementation delivers competitive performance!")
+    print("=" * 70)
+    
+    # Technical notes
+    print("\nNOTES:")
+    print("- dgen-py uses Xoshiro256++ RNG (faster than Numpy's MT19937)")
+    print("- dgen-py leverages multi-threading via Rayon")
+    print("- dgen-py provides zero-copy access via buffer protocol")
+    print("- Numpy's random.bytes is single-threaded")
+    print("- Numpy's random.randint has array creation overhead")
+
+
+if __name__ == "__main__":
+    main()

dgen_py/examples/demo.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Example: Generate data and compress it to verify compression ratio
+"""
+
+import dgen_py
+import sys
+
+
+def test_compression_ratio():
+    """Generate data with specific compression ratio and verify"""
+    try:
+        import zstandard as zstd
+    except ImportError:
+        print("Error: zstandard package required")
+        print("Install with: pip install zstandard")
+        sys.exit(1)
+
+    size = 100 * 1024 * 1024  # 100 MiB
+    target_compress_ratio = 3.0  # 3:1 compression
+
+    print(f"Generating {size / (1024**2):.1f} MiB with {target_compress_ratio}:1 compression ratio...")
+    
+    data = dgen_py.generate_data(size, compress_ratio=target_compress_ratio)
+    
+    print(f"Generated {len(data) / (1024**2):.1f} MiB")
+    print("Compressing with zstd...")
+    
+    compressor = zstd.ZstdCompressor(level=3)
+    compressed = compressor.compress(data)
+    
+    actual_ratio = len(data) / len(compressed)
+    
+    print(f"\nResults:")
+    print(f"  Original:   {len(data) / (1024**2):.2f} MiB")
+    print(f"  Compressed: {len(compressed) / (1024**2):.2f} MiB")
+    print(f"  Ratio:      {actual_ratio:.2f}:1")
+    print(f"  Target:     {target_compress_ratio:.2f}:1")
+    print(f"  Delta:      {abs(actual_ratio - target_compress_ratio) / target_compress_ratio * 100:.1f}%")
+
+
+def test_streaming():
+    """Generate large dataset using streaming API"""
+    size = 1024 * 1024 * 1024  # 1 GiB
+    chunk_size = 4 * 1024 * 1024  # 4 MiB chunks
+
+    print(f"\nGenerating {size / (1024**3):.1f} GiB using streaming API...")
+    
+    gen = dgen_py.StreamingGenerator(
+        size=size,
+        dedup_ratio=2.0,
+        compress_ratio=2.0,
+        numa_mode='force'
+    )
+    
+    buf = bytearray(chunk_size)
+    total = 0
+    chunks = 0
+    
+    while not gen.is_complete():
+        nbytes = gen.fill_chunk(buf)
+        if nbytes == 0:
+            break
+        
+        total += nbytes
+        chunks += 1
+        
+        if chunks % 100 == 0:
+            pct = (total / size) * 100
+            print(f"  Progress: {total / (1024**3):.2f} GiB ({pct:.1f}%)")
+    
+    print(f"\nCompleted:")
+    print(f"  Total: {total / (1024**3):.3f} GiB")
+    print(f"  Chunks: {chunks}")
+    print(f"  Avg chunk size: {total / chunks / (1024**2):.2f} MiB")
+
+
+def show_system_info():
+    """Display NUMA topology information"""
+    print("\nSystem Information:")
+    print("-" * 50)
+    
+    info = dgen_py.get_system_info()
+    if info:
+        print(f"  NUMA nodes:      {info['num_nodes']}")
+        print(f"  Physical cores:  {info['physical_cores']}")
+        print(f"  Logical CPUs:    {info['logical_cpus']}")
+        print(f"  UMA system:      {info['is_uma']}")
+        print(f"  Deployment:      {info['deployment_type']}")
+    else:
+        print("  NUMA info not available on this platform")
+
+
+if __name__ == '__main__':
+    show_system_info()
+    
+    print("\n" + "=" * 50)
+    print("Test 1: Compression Ratio Validation")
+    print("=" * 50)
+    test_compression_ratio()
+    
+    print("\n" + "=" * 50)
+    print("Test 2: Streaming Generation")
+    print("=" * 50)
+    test_streaming()
+    
+    print("\n✓ All tests completed!")

dgen_py/examples/quick_perf_test.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Quick Performance Test
+======================
+
+Fast 30-second test to find optimal settings for your system.
+
+Usage:
+    python quick_perf_test.py
+"""
+
+import time
+import sys
+
+try:
+    import dgen_py
+except ImportError:
+    print("ERROR: dgen-py not installed")
+    print("Run: pip install dgen-py")
+    sys.exit(1)
+
+
+def test_config(name, size, **kwargs):
+    """Test a configuration and return throughput"""
+    start = time.perf_counter()
+    data = dgen_py.generate_data(size, **kwargs)
+    elapsed = time.perf_counter() - start
+    throughput = (size / elapsed) / 1e9
+    return throughput, elapsed
+
+
+def main():
+    print("dgen-py Quick Performance Test")
+    print("=" * 50)
+    
+    # System info
+    info = dgen_py.get_system_info()
+    if info:
+        print(f"\nSystem: {info['num_nodes']} NUMA node(s), {info['logical_cpus']} CPUs")
+        if info['num_nodes'] > 1:
+            print("  → Multi-socket system (NUMA optimizations should help!)")
+        else:
+            print("  → Single-socket system (UMA)")
+    
+    size = 100 * 1024 * 1024  # 100 MiB
+    print(f"\nTest size: 100 MiB per run")
+    print(f"\nRunning tests...")
+    print("-" * 50)
+    
+    results = []
+    
+    # Test 1: Default (auto-detect everything)
+    print("\n1. Default (auto-detect)...", end=" ", flush=True)
+    tp, t = test_config("default", size)
+    print(f"{tp:.2f} GB/s")
+    results.append(("Default (auto)", tp, {}))
+    
+    # Test 2: Force NUMA
+    print("2. Force NUMA...", end=" ", flush=True)
+    tp, t = test_config("force_numa", size, numa_mode="force")
+    print(f"{tp:.2f} GB/s")
+    results.append(("Force NUMA", tp, {"numa_mode": "force"}))
+    
+    # Test 3: NUMA disabled
+    print("3. NUMA disabled...", end=" ", flush=True)
+    tp, t = test_config("disabled_numa", size, numa_mode="disabled")
+    print(f"{tp:.2f} GB/s")
+    results.append(("NUMA disabled", tp, {"numa_mode": "disabled"}))
+    
+    # Test 4: Half threads
+    import multiprocessing
+    half_threads = multiprocessing.cpu_count() // 2
+    print(f"4. Half threads ({half_threads})...", end=" ", flush=True)
+    tp, t = test_config("half_threads", size, max_threads=half_threads)
+    print(f"{tp:.2f} GB/s")
+    results.append((f"Half threads ({half_threads})", tp, {"max_threads": half_threads}))
+    
+    # Test 5: Single thread (baseline)
+    print("5. Single thread (baseline)...", end=" ", flush=True)
+    tp, t = test_config("single", size, max_threads=1)
+    print(f"{tp:.2f} GB/s")
+    results.append(("Single thread", tp, {"max_threads": 1}))
+    
+    # Find best
+    results.sort(key=lambda x: x[1], reverse=True)
+    
+    print("\n" + "=" * 50)
+    print("RESULTS (fastest to slowest):")
+    print("=" * 50)
+    for i, (name, tp, config) in enumerate(results, 1):
+        star = "★" if i == 1 else " "
+        print(f"{star} {i}. {name:25s} {tp:8.2f} GB/s")
+    
+    # Recommendation
+    best_name, best_tp, best_config = results[0]
+    print("\n" + "=" * 50)
+    print(f"RECOMMENDATION: {best_name}")
+    print(f"  Throughput: {best_tp:.2f} GB/s")
+    if best_config:
+        print(f"  Code: dgen_py.generate_data(size, {', '.join(f'{k}={repr(v)}' for k, v in best_config.items())})")
+    else:
+        print(f"  Code: dgen_py.generate_data(size)  # defaults are optimal!")
+    print("=" * 50)
+
+
+if __name__ == "__main__":
+    main()