explodethosebits 0.3.0__cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl

include/etb/cuda/arch_optimizations.cuh

@@ -0,0 +1,358 @@
+#ifndef ETB_ARCH_OPTIMIZATIONS_CUH
+#define ETB_ARCH_OPTIMIZATIONS_CUH
+
+#include "cuda_common.cuh"
+
+namespace etb {
+namespace cuda {
+
+// ============================================================================
+// Architecture Detection and Configuration
+// ============================================================================
+
+/**
+ * Runtime architecture detection.
+ */
+struct ArchitectureInfo {
+    int sm_version;
+    bool is_hopper;         // SM 90
+    bool is_blackwell;      // SM 100
+    bool has_tensor_cores;
+    bool has_async_copy;
+    bool has_cluster_launch;
+    size_t max_shared_mem;
+    int max_threads_per_sm;
+    int registers_per_sm;
+    
+    ArchitectureInfo()
+        : sm_version(0), is_hopper(false), is_blackwell(false)
+        , has_tensor_cores(false), has_async_copy(false)
+        , has_cluster_launch(false), max_shared_mem(0)
+        , max_threads_per_sm(0), registers_per_sm(0) {}
+};
+
+/**
+ * Get architecture information for a device.
+ */
+ArchitectureInfo get_architecture_info(int device_id = 0);
+
+// ============================================================================
+// Hopper (SM 90) Optimizations
+// ============================================================================
+
+namespace hopper {
+
+/**
+ * Hopper-optimized kernel configuration.
+ */
+struct HopperConfig {
+    // Thread block configuration
+    static constexpr int THREADS_PER_BLOCK = 256;
+    static constexpr int WARPS_PER_BLOCK = THREADS_PER_BLOCK / 32;
+    
+    // Shared memory configuration
+    static constexpr size_t DEFAULT_SHARED_MEM = 48 * 1024;  // 48KB
+    static constexpr size_t MAX_SHARED_MEM = 228 * 1024;     // 228KB with opt-in
+    
+    // Occupancy targets
+    static constexpr int TARGET_BLOCKS_PER_SM = 4;
+    static constexpr int REGISTERS_PER_THREAD = 64;
+    
+    // Memory access patterns
+    static constexpr int CACHE_LINE_SIZE = 128;
+    static constexpr int SECTOR_SIZE = 32;
+};
+
+/**
+ * Configure kernel for Hopper architecture.
+ * Enables extended shared memory if beneficial.
+ */
+template<typename KernelFunc>
+void configure_hopper_kernel(KernelFunc kernel, size_t shared_mem_required) {
+    if (shared_mem_required > HopperConfig::DEFAULT_SHARED_MEM) {
+        // Request extended shared memory
+        cudaFuncSetAttribute(kernel, 
+            cudaFuncAttributeMaxDynamicSharedMemorySize,
+            static_cast<int>(HopperConfig::MAX_SHARED_MEM));
+    }
+    
+    // Set preferred cache configuration
+    cudaFuncSetCacheConfig(kernel, cudaFuncCachePreferShared);
+}
+
+/**
+ * Hopper-optimized memory copy using async copy.
+ * Uses cp.async for efficient global to shared memory transfers.
+ * Note: For variable-size copies, use the templated version or memcpy fallback.
+ */
+template<size_t BYTES>
+__device__ inline void async_copy_global_to_shared_fixed(
+    void* shared_dst,
+    const void* global_src
+) {
+#if __CUDA_ARCH__ >= 900
+    // Use cp.async for Hopper with compile-time constant size
+    static_assert(BYTES == 4 || BYTES == 8 || BYTES == 16, 
+                  "cp.async only supports 4, 8, or 16 byte copies");
+    asm volatile(
+        "cp.async.ca.shared.global [%0], [%1], %2;\n"
+        :
+        : "r"(static_cast<unsigned int>(__cvta_generic_to_shared(shared_dst))),
+          "l"(global_src),
+          "n"(BYTES)
+    );
+#else
+    // Fallback for older architectures
+    memcpy(shared_dst, global_src, BYTES);
+#endif
+}
+
+/**
+ * Variable-size async copy (uses memcpy fallback for non-constant sizes).
+ */
+__device__ inline void async_copy_global_to_shared(
+    void* shared_dst,
+    const void* global_src,
+    size_t bytes
+) {
+    // For variable sizes, use standard memcpy
+    // cp.async requires compile-time constant sizes
+    memcpy(shared_dst, global_src, bytes);
+}
+
+/**
+ * Commit async copies and wait.
+ */
+__device__ inline void async_copy_commit_and_wait() {
+#if __CUDA_ARCH__ >= 900
+    asm volatile("cp.async.commit_group;\n");
+    asm volatile("cp.async.wait_group 0;\n");
+#endif
+    __syncthreads();
+}
+
+/**
+ * Hopper-optimized warp-level reduction.
+ * Uses warp shuffle with reduced synchronization.
+ */
+template<typename T>
+__device__ inline T hopper_warp_reduce_sum(T val) {
+    // Hopper supports efficient warp shuffles
+    #pragma unroll
+    for (int offset = 16; offset > 0; offset /= 2) {
+        val += __shfl_down_sync(0xFFFFFFFF, val, offset);
+    }
+    return val;
+}
+
+/**
+ * Hopper-optimized block-level reduction.
+ */
+template<typename T>
+__device__ inline T hopper_block_reduce_sum(T val, T* shared_data) {
+    const int tid = threadIdx.x;
+    const int lane_id = tid % 32;
+    const int warp_id = tid / 32;
+    
+    // Warp-level reduction
+    val = hopper_warp_reduce_sum(val);
+    
+    // Store warp results
+    if (lane_id == 0) {
+        shared_data[warp_id] = val;
+    }
+    __syncthreads();
+    
+    // Final reduction in first warp
+    if (warp_id == 0) {
+        val = (tid < HopperConfig::WARPS_PER_BLOCK) ? shared_data[tid] : T(0);
+        val = hopper_warp_reduce_sum(val);
+    }
+    
+    return val;
+}
+
+} // namespace hopper
+
+// ============================================================================
+// Blackwell (SM 100) Optimizations
+// ============================================================================
+
+namespace blackwell {
+
+/**
+ * Blackwell-optimized kernel configuration.
+ */
+struct BlackwellConfig {
+    // Thread block configuration - Blackwell supports larger blocks
+    static constexpr int THREADS_PER_BLOCK = 512;
+    static constexpr int WARPS_PER_BLOCK = THREADS_PER_BLOCK / 32;
+    
+    // Shared memory configuration - Blackwell has more shared memory
+    static constexpr size_t DEFAULT_SHARED_MEM = 64 * 1024;  // 64KB
+    static constexpr size_t MAX_SHARED_MEM = 256 * 1024;     // 256KB with opt-in
+    
+    // Occupancy targets
+    static constexpr int TARGET_BLOCKS_PER_SM = 2;
+    static constexpr int REGISTERS_PER_THREAD = 128;
+    
+    // Memory access patterns
+    static constexpr int CACHE_LINE_SIZE = 128;
+    static constexpr int SECTOR_SIZE = 32;
+    
+    // Blackwell-specific features
+    static constexpr bool HAS_ENHANCED_TENSOR_CORES = true;
+    static constexpr bool HAS_IMPROVED_L2_CACHE = true;
+};
+
+/**
+ * Configure kernel for Blackwell architecture.
+ */
+template<typename KernelFunc>
+void configure_blackwell_kernel(KernelFunc kernel, size_t shared_mem_required) {
+    if (shared_mem_required > BlackwellConfig::DEFAULT_SHARED_MEM) {
+        cudaFuncSetAttribute(kernel,
+            cudaFuncAttributeMaxDynamicSharedMemorySize,
+            static_cast<int>(BlackwellConfig::MAX_SHARED_MEM));
+    }
+    
+    // Blackwell benefits from L2 cache preference for read-heavy workloads
+    cudaFuncSetCacheConfig(kernel, cudaFuncCachePreferL1);
+}
+
+/**
+ * Blackwell-optimized memory prefetch.
+ * Uses improved prefetch instructions.
+ */
+__device__ inline void prefetch_global(const void* ptr) {
+#if __CUDA_ARCH__ >= 1000
+    // Blackwell prefetch
+    asm volatile("prefetch.global.L2 [%0];\n" : : "l"(ptr));
+#elif __CUDA_ARCH__ >= 900
+    // Hopper prefetch
+    asm volatile("prefetch.global.L2 [%0];\n" : : "l"(ptr));
+#endif
+}
+
+/**
+ * Blackwell-optimized warp reduction with larger register file.
+ */
+template<typename T>
+__device__ inline T blackwell_warp_reduce_sum(T val) {
+    #pragma unroll
+    for (int offset = 16; offset > 0; offset /= 2) {
+        val += __shfl_down_sync(0xFFFFFFFF, val, offset);
+    }
+    return val;
+}
+
+/**
+ * Blackwell-optimized block reduction for larger blocks.
+ */
+template<typename T>
+__device__ inline T blackwell_block_reduce_sum(T val, T* shared_data) {
+    const int tid = threadIdx.x;
+    const int lane_id = tid % 32;
+    const int warp_id = tid / 32;
+    
+    // Warp-level reduction
+    val = blackwell_warp_reduce_sum(val);
+    
+    // Store warp results
+    if (lane_id == 0) {
+        shared_data[warp_id] = val;
+    }
+    __syncthreads();
+    
+    // Final reduction - Blackwell has more warps per block
+    if (warp_id == 0) {
+        val = (tid < BlackwellConfig::WARPS_PER_BLOCK) ? shared_data[tid] : T(0);
+        val = blackwell_warp_reduce_sum(val);
+    }
+    
+    return val;
+}
+
+/**
+ * Blackwell-optimized histogram using larger shared memory.
+ */
+__device__ inline void blackwell_histogram_add(uint32_t* histogram, uint8_t value) {
+    // Blackwell can handle more concurrent atomics efficiently
+    atomicAdd(&histogram[value], 1);
+}
+
+} // namespace blackwell
+
+// ============================================================================
+// Architecture-Adaptive Kernel Launch
+// ============================================================================
+
+/**
+ * Adaptive kernel configuration based on detected architecture.
+ */
+struct AdaptiveKernelConfig {
+    int threads_per_block;
+    int blocks_per_grid;
+    size_t shared_mem_size;
+    bool use_async_copy;
+    bool use_extended_shared_mem;
+    
+    AdaptiveKernelConfig()
+        : threads_per_block(256), blocks_per_grid(1)
+        , shared_mem_size(48 * 1024), use_async_copy(false)
+        , use_extended_shared_mem(false) {}
+};
+
+/**
+ * Get adaptive configuration for current device.
+ */
+AdaptiveKernelConfig get_adaptive_config(int device_id, size_t work_items, 
+                                          size_t shared_mem_required);
+
+/**
+ * Architecture-specific kernel launcher.
+ */
+class AdaptiveKernelLauncher {
+public:
+    AdaptiveKernelLauncher();
+    ~AdaptiveKernelLauncher();
+    
+    /**
+     * Initialize for a specific device.
+     */
+    void initialize(int device_id);
+    
+    /**
+     * Get the detected architecture.
+     */
+    const ArchitectureInfo& get_arch_info() const { return arch_info_; }
+    
+    /**
+     * Check if Hopper optimizations should be used.
+     */
+    bool use_hopper_optimizations() const { return arch_info_.is_hopper; }
+    
+    /**
+     * Check if Blackwell optimizations should be used.
+     */
+    bool use_blackwell_optimizations() const { return arch_info_.is_blackwell; }
+    
+    /**
+     * Get optimal thread count for current architecture.
+     */
+    int get_optimal_threads() const;
+    
+    /**
+     * Get optimal shared memory size for current architecture.
+     */
+    size_t get_optimal_shared_mem() const;
+
+private:
+    ArchitectureInfo arch_info_;
+    bool initialized_;
+};
+
+} // namespace cuda
+} // namespace etb
+
+#endif // ETB_ARCH_OPTIMIZATIONS_CUH

include/etb/cuda/blackwell_optimizations.cuh

@@ -0,0 +1,300 @@
+#ifndef ETB_BLACKWELL_OPTIMIZATIONS_CUH
+#define ETB_BLACKWELL_OPTIMIZATIONS_CUH
+
+#include "cuda_common.cuh"
+#include "arch_optimizations.cuh"
+
+// Note: cooperative_groups.h is not included here to avoid namespace pollution
+// with CCCL on MSVC. The cluster functions below are stubbed out.
+
+namespace etb {
+namespace cuda {
+namespace blackwell {
+
+// ============================================================================
+// Blackwell (SM 100) Specific Optimizations
+// ============================================================================
+
+/**
+ * Blackwell memory hierarchy configuration.
+ * Blackwell has improved L2 cache and memory bandwidth.
+ */
+struct BlackwellMemoryConfig {
+    // L2 cache configuration
+    static constexpr size_t L2_CACHE_SIZE = 96 * 1024 * 1024;  // 96MB typical
+    static constexpr size_t L2_CACHE_LINE = 128;
+    
+    // Memory bandwidth optimization
+    static constexpr int COALESCING_WIDTH = 128;  // bytes
+    static constexpr int OPTIMAL_VECTOR_WIDTH = 4; // float4/int4
+    
+    // Prefetch distances
+    static constexpr int PREFETCH_DISTANCE = 4;   // cache lines ahead
+};
+
+/**
+ * Blackwell-optimized path generator configuration.
+ */
+struct BlackwellPathGeneratorConfig {
+    // Larger thread blocks for better occupancy
+    static constexpr int THREADS_PER_BLOCK = 512;
+    static constexpr int PATHS_PER_THREAD = 4;
+    
+    // Work distribution
+    static constexpr int WORK_ITEMS_PER_BLOCK = THREADS_PER_BLOCK * PATHS_PER_THREAD;
+    
+    // Shared memory layout
+    static constexpr size_t SHARED_PREFIX_CACHE_SIZE = 32 * 1024;  // 32KB for prefix cache
+    static constexpr size_t SHARED_WORK_QUEUE_SIZE = 16 * 1024;    // 16KB for work queue
+};
+
+/**
+ * Blackwell-optimized heuristics configuration.
+ */
+struct BlackwellHeuristicsConfig {
+    // Histogram configuration
+    static constexpr int HISTOGRAM_BANKS = 32;  // Reduce bank conflicts
+    static constexpr int HISTOGRAM_SIZE = 256 * HISTOGRAM_BANKS;
+    
+    // Parallel reduction
+    static constexpr int REDUCTION_THREADS = 512;
+    static constexpr int REDUCTION_WARPS = REDUCTION_THREADS / 32;
+};
+
+/**
+ * Blackwell-optimized vectorized memory load.
+ * Uses float4 for coalesced 128-bit loads.
+ */
+__device__ inline float4 blackwell_load_float4(const float* ptr) {
+    return *reinterpret_cast<const float4*>(ptr);
+}
+
+__device__ inline uint4 blackwell_load_uint4(const uint32_t* ptr) {
+    return *reinterpret_cast<const uint4*>(ptr);
+}
+
+/**
+ * Blackwell-optimized vectorized memory store.
+ */
+__device__ inline void blackwell_store_float4(float* ptr, float4 val) {
+    *reinterpret_cast<float4*>(ptr) = val;
+}
+
+__device__ inline void blackwell_store_uint4(uint32_t* ptr, uint4 val) {
+    *reinterpret_cast<uint4*>(ptr) = val;
+}
+
+/**
+ * Blackwell L2 cache hint for read-only data.
+ */
+__device__ inline void blackwell_cache_hint_readonly(const void* ptr) {
+#if __CUDA_ARCH__ >= 1000
+    asm volatile("prefetch.global.L2::evict_last [%0];\n" : : "l"(ptr));
+#endif
+}
+
+/**
+ * Blackwell L2 cache hint for streaming data.
+ */
+__device__ inline void blackwell_cache_hint_streaming(const void* ptr) {
+#if __CUDA_ARCH__ >= 1000
+    asm volatile("prefetch.global.L2::evict_first [%0];\n" : : "l"(ptr));
+#endif
+}
+
+/**
+ * Blackwell-optimized histogram with bank conflict avoidance.
+ * Uses padding to avoid shared memory bank conflicts.
+ */
+template<int BANKS = 32>
+__device__ inline void blackwell_histogram_add_banked(
+    uint32_t* histogram,  // Size should be 256 * BANKS
+    uint8_t value,
+    int thread_id
+) {
+    // Each thread uses a different bank based on thread ID
+    int bank = thread_id % BANKS;
+    atomicAdd(&histogram[value * BANKS + bank], 1);
+}
+
+/**
+ * Reduce banked histogram to final histogram.
+ */
+template<int BANKS = 32>
+__device__ inline void blackwell_histogram_reduce(
+    uint32_t* banked_histogram,  // Input: 256 * BANKS
+    uint32_t* final_histogram,   // Output: 256
+    int thread_id,
+    int block_size
+) {
+    // Each thread reduces one or more bins
+    for (int bin = thread_id; bin < 256; bin += block_size) {
+        uint32_t sum = 0;
+        for (int bank = 0; bank < BANKS; ++bank) {
+            sum += banked_histogram[bin * BANKS + bank];
+        }
+        final_histogram[bin] = sum;
+    }
+}
+
+/**
+ * Blackwell-optimized parallel entropy calculation.
+ * Uses larger thread blocks and vectorized operations.
+ */
+__device__ inline float blackwell_calculate_entropy(
+    const uint32_t* histogram,
+    uint32_t total,
+    float* scratch,
+    int tid,
+    int block_size
+) {
+    // Each thread handles multiple bins
+    int bins_per_thread = (256 + block_size - 1) / block_size;
+    float local_entropy = 0.0f;
+    
+    #pragma unroll 4
+    for (int i = 0; i < bins_per_thread; ++i) {
+        int bin = tid * bins_per_thread + i;
+        if (bin < 256) {
+            uint32_t count = histogram[bin];
+            if (count > 0 && total > 0) {
+                float p = static_cast<float>(count) / static_cast<float>(total);
+                local_entropy -= p * log2f(p);
+            }
+        }
+    }
+    
+    // Block reduction
+    scratch[tid] = local_entropy;
+    __syncthreads();
+    
+    // Tree reduction with larger stride for 512 threads
+    for (int stride = block_size / 2; stride > 0; stride /= 2) {
+        if (tid < stride) {
+            scratch[tid] += scratch[tid + stride];
+        }
+        __syncthreads();
+    }
+    
+    return scratch[0];
+}
+
+/**
+ * Blackwell-optimized signature matching with vectorized comparison.
+ */
+__device__ inline bool blackwell_signature_match_vectorized(
+    const uint8_t* data,
+    const uint8_t* signature,
+    const uint8_t* mask,
+    int length
+) {
+    // Process 4 bytes at a time when possible
+    int vec_length = length / 4;
+    int remainder = length % 4;
+    
+    // Vectorized comparison
+    for (int i = 0; i < vec_length; ++i) {
+        uint32_t data_vec = *reinterpret_cast<const uint32_t*>(data + i * 4);
+        uint32_t sig_vec = *reinterpret_cast<const uint32_t*>(signature + i * 4);
+        uint32_t mask_vec = *reinterpret_cast<const uint32_t*>(mask + i * 4);
+        
+        if ((data_vec & mask_vec) != (sig_vec & mask_vec)) {
+            return false;
+        }
+    }
+    
+    // Handle remainder
+    for (int i = vec_length * 4; i < length; ++i) {
+        if ((data[i] & mask[i]) != (signature[i] & mask[i])) {
+            return false;
+        }
+    }
+    
+    return true;
+}
+
+/**
+ * Blackwell-optimized work stealing with improved atomics.
+ */
+__device__ inline bool blackwell_steal_work(
+    uint32_t* work_queue,
+    uint32_t* head,
+    uint32_t* tail,
+    uint32_t queue_capacity,
+    uint32_t& work_item
+) {
+    // Use atomic exchange for more efficient stealing
+    uint32_t old_head = atomicAdd(head, 1);
+    uint32_t current_tail = __ldcg(tail);  // Cache-global load
+    
+    if (old_head < current_tail) {
+        work_item = work_queue[old_head % queue_capacity];
+        return true;
+    }
+    
+    // Restore head if no work
+    atomicSub(head, 1);
+    return false;
+}
+
+/**
+ * Blackwell-optimized cooperative group operations.
+ * Uses thread block clusters when available.
+ * Note: These are stubbed out due to CCCL/MSVC compatibility issues.
+ * Full implementation would require cooperative_groups.h.
+ */
+#if __CUDA_ARCH__ >= 1000
+__device__ inline void blackwell_cluster_sync() {
+    // Stub - would use cooperative_groups cluster sync
+    __syncthreads();
+}
+
+__device__ inline int blackwell_cluster_thread_rank() {
+    // Stub - returns block-local thread rank
+    return threadIdx.x + threadIdx.y * blockDim.x + threadIdx.z * blockDim.x * blockDim.y;
+}
+
+__device__ inline int blackwell_cluster_size() {
+    // Stub - returns block size
+    return blockDim.x * blockDim.y * blockDim.z;
+}
+#endif
+
+/**
+ * Configure kernel for Blackwell with cluster launch.
+ */
+template<typename KernelFunc>
+cudaError_t configure_blackwell_cluster_kernel(
+    KernelFunc kernel,
+    int cluster_size,
+    size_t shared_mem_required
+) {
+    cudaError_t err;
+    
+    // Set max dynamic shared memory
+    if (shared_mem_required > BlackwellConfig::DEFAULT_SHARED_MEM) {
+        err = cudaFuncSetAttribute(kernel,
+            cudaFuncAttributeMaxDynamicSharedMemorySize,
+            static_cast<int>(BlackwellConfig::MAX_SHARED_MEM));
+        if (err != cudaSuccess) return err;
+    }
+    
+    // Set cluster dimensions (Blackwell feature)
+#if CUDART_VERSION >= 12000
+    cudaFuncAttribute attr = cudaFuncAttributeClusterDimMustBeSet;
+    err = cudaFuncSetAttribute(kernel, attr, 1);
+    if (err != cudaSuccess) return err;
+    
+    attr = cudaFuncAttributeClusterSchedulingPolicyPreference;
+    err = cudaFuncSetAttribute(kernel, attr, 
+        cudaClusterSchedulingPolicySpread);
+#endif
+    
+    return cudaSuccess;
+}
+
+} // namespace blackwell
+} // namespace cuda
+} // namespace etb
+
+#endif // ETB_BLACKWELL_OPTIMIZATIONS_CUH