explodethosebits 0.3.0__cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl

include/etb/cuda/signature_kernel.cuh

@@ -0,0 +1,328 @@
+#ifndef ETB_SIGNATURE_KERNEL_CUH
+#define ETB_SIGNATURE_KERNEL_CUH
+
+#include "cuda_common.cuh"
+#include "gpu_memory.cuh"
+
+namespace etb {
+namespace cuda {
+
+// Constant memory declarations (defined in gpu_memory.cu)
+extern __constant__ DeviceFileSignature d_signatures[MAX_SIGNATURES];
+extern __constant__ DeviceFooterSignature d_footers[MAX_SIGNATURES];
+extern __constant__ uint32_t d_signature_count;
+extern __constant__ uint32_t d_footer_count;
+
+/**
+ * Shared memory structure for signature matching.
+ */
+struct SignatureSharedMem {
+    // Best match found by this block
+    DeviceSignatureMatch best_match;
+    
+    // Reduction scratch space
+    float match_scores[256];
+    uint32_t match_indices[256];
+};
+
+/**
+ * Signature matcher CUDA kernel.
+ * 
+ * Performs parallel sliding window matching using constant memory
+ * for signature broadcast.
+ * 
+ * Requirements: 9.4
+ * 
+ * @param data Input byte data
+ * @param length Length of data
+ * @param max_offset Maximum offset to search for headers
+ * @param result Output signature match result
+ */
+__global__ void signature_matcher_kernel(
+    const uint8_t* data,
+    uint32_t length,
+    uint32_t max_offset,
+    DeviceSignatureMatch* result
+);
+
+/**
+ * Batch signature matching kernel.
+ * Each block handles one byte sequence.
+ * 
+ * @param data_ptrs Array of pointers to byte sequences
+ * @param lengths Array of sequence lengths
+ * @param num_sequences Number of sequences
+ * @param max_offset Maximum offset to search
+ * @param results Output array of match results
+ */
+__global__ void batch_signature_matcher_kernel(
+    const uint8_t** data_ptrs,
+    const uint32_t* lengths,
+    uint32_t num_sequences,
+    uint32_t max_offset,
+    DeviceSignatureMatch* results
+);
+
+/**
+ * Quick signature prefix check kernel.
+ * Checks only the first few bytes for fast rejection.
+ * 
+ * @param data Input byte data
+ * @param length Length of data
+ * @param has_potential_match Output flag indicating if any signature could match
+ */
+__global__ void signature_prefix_check_kernel(
+    const uint8_t* data,
+    uint32_t length,
+    bool* has_potential_match
+);
+
+/**
+ * Host-side launcher for signature matching kernels.
+ */
+class SignatureMatcherKernel {
+public:
+    SignatureMatcherKernel();
+    ~SignatureMatcherKernel();
+    
+    /**
+     * Configure the kernel for a specific device.
+     * @param device_id CUDA device ID
+     */
+    void configure(int device_id);
+    
+    /**
+     * Match signatures against a byte sequence.
+     * @param data Device pointer to byte data
+     * @param length Length of data
+     * @param max_offset Maximum offset to search
+     * @param result Device pointer to result
+     * @param stream CUDA stream
+     */
+    void match(const uint8_t* data, uint32_t length, uint32_t max_offset,
+               DeviceSignatureMatch* result, cudaStream_t stream = nullptr);
+    
+    /**
+     * Match signatures against multiple byte sequences.
+     * @param data_ptrs Device array of pointers
+     * @param lengths Device array of lengths
+     * @param num_sequences Number of sequences
+     * @param max_offset Maximum offset to search
+     * @param results Device array of results
+     * @param stream CUDA stream
+     */
+    void match_batch(const uint8_t** data_ptrs, const uint32_t* lengths,
+                     uint32_t num_sequences, uint32_t max_offset,
+                     DeviceSignatureMatch* results, cudaStream_t stream = nullptr);
+    
+    /**
+     * Quick prefix check for early rejection.
+     * @param data Device pointer to byte data
+     * @param length Length of data
+     * @param has_potential Device pointer to result flag
+     * @param stream CUDA stream
+     */
+    void prefix_check(const uint8_t* data, uint32_t length,
+                      bool* has_potential, cudaStream_t stream = nullptr);
+    
+    /**
+     * Get the kernel configuration.
+     */
+    const KernelConfig& get_config() const { return kernel_config_; }
+
+private:
+    KernelConfig kernel_config_;
+    bool configured_;
+};
+
+// ============================================================================
+// Device Functions
+// ============================================================================
+
+/**
+ * Check if signature matches at a specific position.
+ * Uses constant memory for signature data.
+ */
+__device__ inline bool check_signature_at_position(
+    const uint8_t* data,
+    uint32_t data_length,
+    uint32_t position,
+    uint32_t sig_idx
+) {
+    const DeviceFileSignature& sig = d_signatures[sig_idx];
+    
+    // Check if signature fits at this position
+    if (position + sig.offset + sig.length > data_length) {
+        return false;
+    }
+    
+    // Compare bytes with mask
+    const uint8_t* check_pos = data + position + sig.offset;
+    for (uint8_t i = 0; i < sig.length; ++i) {
+        uint8_t masked_data = check_pos[i] & sig.mask[i];
+        uint8_t masked_sig = sig.magic_bytes[i] & sig.mask[i];
+        if (masked_data != masked_sig) {
+            return false;
+        }
+    }
+    
+    return true;
+}
+
+/**
+ * Check if footer matches at end of data.
+ */
+__device__ inline bool check_footer_at_end(
+    const uint8_t* data,
+    uint32_t data_length,
+    uint32_t footer_idx
+) {
+    const DeviceFooterSignature& footer = d_footers[footer_idx];
+    
+    if (footer.length == 0 || footer.length > data_length) {
+        return false;
+    }
+    
+    const uint8_t* footer_pos = data + data_length - footer.length;
+    for (uint8_t i = 0; i < footer.length; ++i) {
+        if (footer_pos[i] != footer.magic_bytes[i]) {
+            return false;
+        }
+    }
+    
+    return true;
+}
+
+/**
+ * Calculate match confidence based on match quality.
+ */
+__device__ inline float calculate_match_confidence(
+    const DeviceFileSignature& sig,
+    bool header_matched,
+    bool footer_matched,
+    bool footer_required
+) {
+    float confidence = sig.base_confidence;
+    
+    if (!header_matched) {
+        return 0.0f;
+    }
+    
+    if (footer_matched) {
+        // Boost confidence for header + footer match
+        confidence = fminf(confidence + 0.1f, 1.0f);
+    } else if (footer_required) {
+        // Reduce confidence if footer was required but not found
+        confidence *= 0.7f;
+    }
+    
+    return confidence;
+}
+
+/**
+ * Warp-level reduction to find best match.
+ */
+__device__ inline void warp_reduce_best_match(
+    float& best_score,
+    uint32_t& best_idx,
+    uint32_t& best_offset,
+    bool& best_header,
+    bool& best_footer
+) {
+    for (int offset = 16; offset > 0; offset /= 2) {
+        float other_score = __shfl_down_sync(0xFFFFFFFF, best_score, offset);
+        uint32_t other_idx = __shfl_down_sync(0xFFFFFFFF, best_idx, offset);
+        uint32_t other_offset = __shfl_down_sync(0xFFFFFFFF, best_offset, offset);
+        bool other_header = __shfl_down_sync(0xFFFFFFFF, best_header ? 1 : 0, offset);
+        bool other_footer = __shfl_down_sync(0xFFFFFFFF, best_footer ? 1 : 0, offset);
+        
+        if (other_score > best_score) {
+            best_score = other_score;
+            best_idx = other_idx;
+            best_offset = other_offset;
+            best_header = other_header;
+            best_footer = other_footer;
+        }
+    }
+}
+
+/**
+ * Inline signature check for use within other kernels.
+ * Checks first 4 bytes against all signatures for quick rejection.
+ */
+__device__ inline bool quick_signature_check(const uint8_t* data, uint32_t length) {
+    if (length < 4) return false;
+    
+    uint32_t sig_count = d_signature_count;
+    
+    for (uint32_t i = 0; i < sig_count; ++i) {
+        const DeviceFileSignature& sig = d_signatures[i];
+        
+        // Only check first 4 bytes for quick match
+        uint8_t check_len = sig.length < 4 ? sig.length : 4;
+        bool matches = true;
+        
+        for (uint8_t j = 0; j < check_len && matches; ++j) {
+            uint8_t masked_data = data[j] & sig.mask[j];
+            uint8_t masked_sig = sig.magic_bytes[j] & sig.mask[j];
+            if (masked_data != masked_sig) {
+                matches = false;
+            }
+        }
+        
+        if (matches) return true;
+    }
+    
+    return false;
+}
+
+/**
+ * Full signature match for inline use.
+ * Returns the best matching signature index or -1 if no match.
+ */
+__device__ inline DeviceSignatureMatch inline_signature_match(
+    const uint8_t* data,
+    uint32_t length,
+    uint32_t max_offset
+) {
+    DeviceSignatureMatch result;
+    result.matched = false;
+    result.confidence = 0.0f;
+    
+    uint32_t sig_count = d_signature_count;
+    
+    for (uint32_t offset = 0; offset <= max_offset && offset < length; ++offset) {
+        for (uint32_t i = 0; i < sig_count; ++i) {
+            if (check_signature_at_position(data, length, offset, i)) {
+                const DeviceFileSignature& sig = d_signatures[i];
+                
+                // Check footer if available
+                bool footer_matched = false;
+                bool footer_required = false;
+                if (i < d_footer_count) {
+                    footer_required = d_footers[i].required;
+                    footer_matched = check_footer_at_end(data, length, i);
+                }
+                
+                float confidence = calculate_match_confidence(
+                    sig, true, footer_matched, footer_required);
+                
+                if (confidence > result.confidence) {
+                    result.matched = true;
+                    result.format_id = sig.format_id;
+                    result.confidence = confidence;
+                    result.match_offset = offset;
+                    result.header_matched = true;
+                    result.footer_matched = footer_matched;
+                }
+            }
+        }
+    }
+    
+    return result;
+}
+
+} // namespace cuda
+} // namespace etb
+
+#endif // ETB_SIGNATURE_KERNEL_CUH

include/etb/early_stopping.hpp

@@ -0,0 +1,246 @@
+#ifndef ETB_EARLY_STOPPING_HPP
+#define ETB_EARLY_STOPPING_HPP
+
+#include <cstdint>
+#include <cstddef>
+#include <vector>
+#include "etb/heuristics.hpp"
+#include "etb/signature.hpp"
+
+namespace etb {
+
+/**
+ * Stop levels for multi-level early stopping.
+ * Each level represents a depth at which stopping decisions are made.
+ */
+enum class StopLevel : uint8_t {
+    LEVEL_1 = 4,    // 2-4 bytes: signature prefix + basic heuristics
+    LEVEL_2 = 8,    // 8 bytes: entropy bounds + checksum validation
+    LEVEL_3 = 16    // 16 bytes: structural coherence
+};
+
+/**
+ * Result of an early stopping check.
+ */
+struct StopDecision {
+    bool should_stop;           // Whether to stop exploring this path
+    StopLevel level;            // Level at which decision was made
+    float score;                // Heuristic score at decision point
+    const char* reason;         // Human-readable reason for stopping
+
+    StopDecision() 
+        : should_stop(false), level(StopLevel::LEVEL_1), 
+          score(0.0f), reason(nullptr) {}
+};
+
+/**
+ * Configuration for early stopping thresholds.
+ */
+struct EarlyStoppingConfig {
+    // Byte thresholds for each level
+    uint32_t level1_bytes;      // Default: 4
+    uint32_t level2_bytes;      // Default: 8
+    uint32_t level3_bytes;      // Default: 16
+
+    // Entropy bounds
+    float min_entropy;          // Below this = repeated pattern garbage (default: 0.1)
+    float max_entropy;          // Above this = random/encrypted (default: 7.9)
+
+    // Heuristic thresholds
+    float level1_threshold;     // Minimum score to continue at level 1 (default: 0.2)
+    float level2_threshold;     // Minimum score to continue at level 2 (default: 0.3)
+    float level3_threshold;     // Minimum score to continue at level 3 (default: 0.4)
+
+    // Adaptive threshold settings
+    bool adaptive_thresholds;   // Enable adaptive threshold adjustment
+    float adaptive_tighten;     // Threshold when global best > 0.8 (default: 0.6)
+    float adaptive_relax;       // Threshold when global best < 0.3 (default: 0.2)
+    float adaptive_default;     // Default threshold (default: 0.4)
+
+    EarlyStoppingConfig()
+        : level1_bytes(4)
+        , level2_bytes(8)
+        , level3_bytes(16)
+        , min_entropy(0.1f)
+        , max_entropy(7.9f)
+        , level1_threshold(0.2f)
+        , level2_threshold(0.3f)
+        , level3_threshold(0.4f)
+        , adaptive_thresholds(true)
+        , adaptive_tighten(0.6f)
+        , adaptive_relax(0.2f)
+        , adaptive_default(0.4f) {}
+};
+
+
+/**
+ * Adaptive threshold manager for early stopping.
+ * Tracks global best score and adjusts thresholds based on running statistics.
+ */
+class AdaptiveThresholdManager {
+public:
+    AdaptiveThresholdManager();
+    explicit AdaptiveThresholdManager(const EarlyStoppingConfig& config);
+
+    /**
+     * Update the global best score seen so far.
+     * @param score New score to consider
+     */
+    void update_best_score(float score);
+
+    /**
+     * Get the current global best score.
+     */
+    float get_best_score() const { return global_best_score_; }
+
+    /**
+     * Get the current adaptive threshold based on global best score.
+     */
+    float get_adaptive_threshold() const;
+
+    /**
+     * Reset the manager to initial state.
+     */
+    void reset();
+
+    /**
+     * Get statistics about threshold adjustments.
+     */
+    uint64_t get_update_count() const { return update_count_; }
+
+private:
+    EarlyStoppingConfig config_;
+    float global_best_score_;
+    uint64_t update_count_;
+};
+
+/**
+ * Early Stopping Controller - CPU Reference Implementation
+ * 
+ * Implements multi-level early stopping to reduce search space from O(8^n) to O(8^d)
+ * where d << n. Uses signature matching, heuristics, and adaptive thresholds.
+ */
+class EarlyStoppingController {
+public:
+    /**
+     * Construct with default configuration.
+     */
+    EarlyStoppingController();
+
+    /**
+     * Construct with custom configuration.
+     * @param config Early stopping configuration
+     */
+    explicit EarlyStoppingController(const EarlyStoppingConfig& config);
+
+    /**
+     * Construct with configuration and signature dictionary.
+     * @param config Early stopping configuration
+     * @param dictionary Signature dictionary for prefix matching
+     */
+    EarlyStoppingController(const EarlyStoppingConfig& config,
+                           const SignatureDictionary* dictionary);
+
+    /**
+     * Set the signature dictionary for prefix matching.
+     * @param dictionary Pointer to signature dictionary (can be nullptr)
+     */
+    void set_signature_dictionary(const SignatureDictionary* dictionary);
+
+    /**
+     * Set the heuristics engine for scoring.
+     * @param engine Pointer to heuristics engine (can be nullptr for default)
+     */
+    void set_heuristics_engine(const HeuristicsEngine* engine);
+
+    /**
+     * Check if a path should be stopped at the current depth.
+     * @param data Reconstructed byte sequence
+     * @param length Length of the byte sequence
+     * @return StopDecision indicating whether to stop and why
+     */
+    StopDecision should_stop(const uint8_t* data, size_t length) const;
+
+    /**
+     * Check if a path should be stopped (vector overload).
+     * @param data Reconstructed byte sequence
+     * @return StopDecision indicating whether to stop and why
+     */
+    StopDecision should_stop(const std::vector<uint8_t>& data) const;
+
+    /**
+     * Check if data consists entirely of repeated bytes.
+     * @param data Byte sequence to check
+     * @param length Length of the sequence
+     * @return true if all bytes are the same value
+     */
+    static bool is_repeated_byte_pattern(const uint8_t* data, size_t length);
+
+    /**
+     * Check if data consists entirely of null bytes.
+     * @param data Byte sequence to check
+     * @param length Length of the sequence
+     * @return true if all bytes are 0x00
+     */
+    static bool is_all_null(const uint8_t* data, size_t length);
+
+    /**
+     * Update the adaptive threshold manager with a new score.
+     * @param score Score to update with
+     */
+    void update_best_score(float score);
+
+    /**
+     * Get the current adaptive threshold.
+     */
+    float get_adaptive_threshold() const;
+
+    /**
+     * Get the configuration.
+     */
+    const EarlyStoppingConfig& get_config() const { return config_; }
+
+    /**
+     * Get statistics about stopping decisions.
+     */
+    struct Statistics {
+        uint64_t total_checks;
+        uint64_t stopped_level1;
+        uint64_t stopped_level2;
+        uint64_t stopped_level3;
+        uint64_t stopped_repeated;
+        uint64_t continued;
+
+        Statistics() : total_checks(0), stopped_level1(0), stopped_level2(0),
+                      stopped_level3(0), stopped_repeated(0), continued(0) {}
+    };
+
+    const Statistics& get_statistics() const { return stats_; }
+    void reset_statistics();
+
+private:
+    EarlyStoppingConfig config_;
+    const SignatureDictionary* dictionary_;
+    const HeuristicsEngine* heuristics_engine_;
+    HeuristicsEngine default_heuristics_;
+    AdaptiveThresholdManager threshold_manager_;
+    mutable Statistics stats_;
+
+    // Level-specific checks
+    StopDecision check_level1(const uint8_t* data, size_t length, 
+                              const HeuristicResult& heuristics) const;
+    StopDecision check_level2(const uint8_t* data, size_t length,
+                              const HeuristicResult& heuristics) const;
+    StopDecision check_level3(const uint8_t* data, size_t length,
+                              const HeuristicResult& heuristics) const;
+
+    // Check for signature prefix match
+    bool has_signature_prefix_match(const uint8_t* data, size_t length) const;
+
+    // Get the effective threshold for a given level
+    float get_threshold_for_level(StopLevel level) const;
+};
+
+} // namespace etb
+
+#endif // ETB_EARLY_STOPPING_HPP

include/etb/etb.hpp

@@ -0,0 +1,20 @@
+#ifndef ETB_HPP
+#define ETB_HPP
+
+// Main include header for ExplodeThoseBits library
+#include "bit_coordinate.hpp"
+#include "path.hpp"
+#include "bit_extraction.hpp"
+#include "path_generator.hpp"
+#include "path_count.hpp"
+#include "signature.hpp"
+#include "heuristics.hpp"
+#include "early_stopping.hpp"
+#include "prefix_trie.hpp"
+#include "memoization.hpp"
+#include "bit_pruning.hpp"
+#include "scoring.hpp"
+#include "config.hpp"
+#include "reporting.hpp"
+
+#endif // ETB_HPP