explodethosebits 0.3.0__cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl

include/etb/heuristics.hpp

@@ -0,0 +1,165 @@
+#ifndef ETB_HEURISTICS_HPP
+#define ETB_HEURISTICS_HPP
+
+#include <cstdint>
+#include <cstddef>
+#include <vector>
+#include <array>
+
+namespace etb {
+
+/**
+ * Result of heuristic analysis on a byte sequence.
+ * All scores are normalized to specific ranges as documented.
+ */
+struct HeuristicResult {
+    float entropy;              // Shannon entropy [0.0, 8.0]
+    float printable_ratio;      // Ratio of printable ASCII [0.0, 1.0]
+    float control_char_ratio;   // Ratio of control characters [0.0, 1.0]
+    uint32_t max_null_run;      // Longest consecutive null byte run
+    float utf8_validity;        // UTF-8 sequence validity score [0.0, 1.0]
+    float composite_score;      // Weighted combination [0.0, 1.0]
+
+    HeuristicResult()
+        : entropy(0.0f)
+        , printable_ratio(0.0f)
+        , control_char_ratio(0.0f)
+        , max_null_run(0)
+        , utf8_validity(0.0f)
+        , composite_score(0.0f) {}
+};
+
+/**
+ * Configurable weights for composite heuristic scoring.
+ */
+struct HeuristicWeights {
+    float entropy_weight;           // Weight for entropy score
+    float printable_weight;         // Weight for printable ratio
+    float control_char_weight;      // Weight for control char penalty
+    float null_run_weight;          // Weight for null run penalty
+    float utf8_weight;              // Weight for UTF-8 validity
+
+    HeuristicWeights()
+        : entropy_weight(0.25f)
+        , printable_weight(0.25f)
+        , control_char_weight(0.15f)
+        , null_run_weight(0.15f)
+        , utf8_weight(0.20f) {}
+};
+
+/**
+ * Heuristics Engine - CPU Reference Implementation
+ * 
+ * Provides functions for analyzing byte sequences to determine viability
+ * as valid data. Used for scoring partial reconstructions during path exploration.
+ */
+class HeuristicsEngine {
+public:
+    /**
+     * Construct a heuristics engine with default weights.
+     */
+    HeuristicsEngine();
+
+    /**
+     * Construct a heuristics engine with custom weights.
+     * @param weights Custom scoring weights
+     */
+    explicit HeuristicsEngine(const HeuristicWeights& weights);
+
+    /**
+     * Set the scoring weights.
+     * @param weights New weights to use
+     */
+    void set_weights(const HeuristicWeights& weights);
+
+    /**
+     * Get the current scoring weights.
+     */
+    const HeuristicWeights& get_weights() const { return weights_; }
+
+    /**
+     * Perform full heuristic analysis on a byte sequence.
+     * @param data Pointer to byte data
+     * @param length Length of data
+     * @return Complete heuristic analysis result
+     */
+    HeuristicResult analyze(const uint8_t* data, size_t length) const;
+
+    /**
+     * Perform full heuristic analysis on a vector of bytes.
+     * @param data Byte vector to analyze
+     * @return Complete heuristic analysis result
+     */
+    HeuristicResult analyze(const std::vector<uint8_t>& data) const;
+
+    // Individual heuristic calculations
+
+    /**
+     * Calculate Shannon entropy of a byte sequence.
+     * @param data Pointer to byte data
+     * @param length Length of data
+     * @return Entropy value in range [0.0, 8.0]
+     */
+    static float calculate_entropy(const uint8_t* data, size_t length);
+
+    /**
+     * Calculate Shannon entropy of a byte vector.
+     * @param data Byte vector
+     * @return Entropy value in range [0.0, 8.0]
+     */
+    static float calculate_entropy(const std::vector<uint8_t>& data);
+
+    /**
+     * Calculate the ratio of printable ASCII characters.
+     * Printable ASCII: bytes in range [0x20, 0x7E]
+     * @param data Pointer to byte data
+     * @param length Length of data
+     * @return Ratio in range [0.0, 1.0]
+     */
+    static float calculate_printable_ratio(const uint8_t* data, size_t length);
+
+    /**
+     * Calculate the ratio of control characters.
+     * Control characters: bytes in range [0x00, 0x1F] excluding common whitespace (0x09, 0x0A, 0x0D)
+     * @param data Pointer to byte data
+     * @param length Length of data
+     * @return Ratio in range [0.0, 1.0]
+     */
+    static float calculate_control_char_ratio(const uint8_t* data, size_t length);
+
+    /**
+     * Find the longest consecutive run of null bytes (0x00).
+     * @param data Pointer to byte data
+     * @param length Length of data
+     * @return Length of longest null byte run
+     */
+    static uint32_t find_max_null_run(const uint8_t* data, size_t length);
+
+    /**
+     * Validate UTF-8 sequences and return a validity score.
+     * @param data Pointer to byte data
+     * @param length Length of data
+     * @return Validity score in range [0.0, 1.0] where 1.0 = fully valid UTF-8
+     */
+    static float validate_utf8(const uint8_t* data, size_t length);
+
+private:
+    HeuristicWeights weights_;
+
+    /**
+     * Build a byte frequency histogram.
+     * @param data Pointer to byte data
+     * @param length Length of data
+     * @return Array of 256 frequency counts
+     */
+    static std::array<uint32_t, 256> build_histogram(const uint8_t* data, size_t length);
+
+    /**
+     * Calculate composite score from individual heuristics.
+     */
+    float calculate_composite_score(const HeuristicResult& result, size_t data_length) const;
+};
+
+} // namespace etb
+
+#endif // ETB_HEURISTICS_HPP

include/etb/memoization.hpp

@@ -0,0 +1,285 @@
+#ifndef ETB_MEMOIZATION_HPP
+#define ETB_MEMOIZATION_HPP
+
+#include <cstdint>
+#include <cstddef>
+#include <vector>
+#include <unordered_map>
+#include <list>
+#include <mutex>
+#include <optional>
+#include "heuristics.hpp"
+
+namespace etb {
+
+/**
+ * Result stored in the prefix cache.
+ * Contains the evaluation result for a specific prefix.
+ */
+struct PrefixCacheEntry {
+    std::vector<uint8_t> prefix;    // The prefix bytes
+    HeuristicResult heuristics;     // Heuristic evaluation result
+    float score;                    // Composite score
+    bool should_prune;              // Whether this prefix should be pruned
+    uint64_t access_count;          // Number of times accessed
+
+    PrefixCacheEntry()
+        : score(0.0f)
+        , should_prune(false)
+        , access_count(0) {}
+
+    PrefixCacheEntry(const std::vector<uint8_t>& p, const HeuristicResult& h, 
+                     float s, bool prune)
+        : prefix(p)
+        , heuristics(h)
+        , score(s)
+        , should_prune(prune)
+        , access_count(1) {}
+};
+
+/**
+ * Configuration for the memoization cache.
+ */
+struct MemoizationConfig {
+    size_t max_size_bytes;          // Maximum cache size in bytes (default: 1GB)
+    size_t max_entries;             // Maximum number of entries (default: 1M)
+    bool enabled;                   // Whether caching is enabled
+
+    MemoizationConfig()
+        : max_size_bytes(1024 * 1024 * 1024)  // 1GB
+        , max_entries(1000000)                 // 1M entries
+        , enabled(true) {}
+};
+
+/**
+ * Statistics for cache operations.
+ */
+struct MemoizationStats {
+    uint64_t hits;                  // Number of cache hits
+    uint64_t misses;                // Number of cache misses
+    uint64_t insertions;            // Number of insertions
+    uint64_t evictions;             // Number of evictions
+    size_t current_entries;         // Current number of entries
+    size_t current_size_bytes;      // Current estimated size in bytes
+
+    MemoizationStats()
+        : hits(0)
+        , misses(0)
+        , insertions(0)
+        , evictions(0)
+        , current_entries(0)
+        , current_size_bytes(0) {}
+
+    void reset() {
+        hits = 0;
+        misses = 0;
+        insertions = 0;
+        evictions = 0;
+        // Don't reset current_entries and current_size_bytes
+    }
+
+    /**
+     * Calculate cache hit rate.
+     * @return Hit rate in range [0.0, 1.0], or 0.0 if no accesses
+     */
+    float hit_rate() const {
+        uint64_t total = hits + misses;
+        if (total == 0) return 0.0f;
+        return static_cast<float>(hits) / static_cast<float>(total);
+    }
+};
+
+/**
+ * Prefix Result Cache with LRU Eviction
+ * 
+ * Stores evaluated prefix results for O(1) lookup on repeated evaluations.
+ * Uses LRU (Least Recently Used) eviction policy when cache size exceeds limits.
+ * 
+ * Requirements: 6.1, 6.2, 6.4, 6.5
+ */
+class PrefixCache {
+public:
+    /**
+     * Construct with default configuration.
+     */
+    PrefixCache();
+
+    /**
+     * Construct with custom configuration.
+     * @param config Cache configuration
+     */
+    explicit PrefixCache(const MemoizationConfig& config);
+
+    /**
+     * Look up a prefix in the cache.
+     * Updates LRU order on hit.
+     * @param prefix Byte sequence representing the prefix
+     * @param length Length of the prefix
+     * @return Optional containing the cached entry if found
+     */
+    std::optional<PrefixCacheEntry> lookup(const uint8_t* prefix, size_t length);
+
+    /**
+     * Look up a prefix in the cache (vector overload).
+     * @param prefix Byte sequence representing the prefix
+     * @return Optional containing the cached entry if found
+     */
+    std::optional<PrefixCacheEntry> lookup(const std::vector<uint8_t>& prefix);
+
+    /**
+     * Insert or update a prefix result in the cache.
+     * May trigger eviction if cache is full.
+     * @param prefix Byte sequence representing the prefix
+     * @param length Length of the prefix
+     * @param heuristics Heuristic evaluation result
+     * @param score Composite score
+     * @param should_prune Whether this prefix should be pruned
+     * @return true if insertion succeeded
+     */
+    bool insert(const uint8_t* prefix, size_t length,
+                const HeuristicResult& heuristics, float score, bool should_prune);
+
+    /**
+     * Insert or update a prefix result in the cache (vector overload).
+     * @param prefix Byte sequence representing the prefix
+     * @param heuristics Heuristic evaluation result
+     * @param score Composite score
+     * @param should_prune Whether this prefix should be pruned
+     * @return true if insertion succeeded
+     */
+    bool insert(const std::vector<uint8_t>& prefix,
+                const HeuristicResult& heuristics, float score, bool should_prune);
+
+    /**
+     * Check if a prefix exists in the cache without updating LRU order.
+     * @param prefix Byte sequence representing the prefix
+     * @param length Length of the prefix
+     * @return true if prefix is cached
+     */
+    bool contains(const uint8_t* prefix, size_t length) const;
+
+    /**
+     * Check if a prefix exists in the cache (vector overload).
+     * @param prefix Byte sequence representing the prefix
+     * @return true if prefix is cached
+     */
+    bool contains(const std::vector<uint8_t>& prefix) const;
+
+    /**
+     * Remove a specific prefix from the cache.
+     * @param prefix Byte sequence representing the prefix
+     * @param length Length of the prefix
+     * @return true if prefix was removed
+     */
+    bool remove(const uint8_t* prefix, size_t length);
+
+    /**
+     * Remove a specific prefix from the cache (vector overload).
+     * @param prefix Byte sequence representing the prefix
+     * @return true if prefix was removed
+     */
+    bool remove(const std::vector<uint8_t>& prefix);
+
+    /**
+     * Clear all entries from the cache.
+     */
+    void clear();
+
+    /**
+     * Get the current number of entries in the cache.
+     */
+    size_t size() const;
+
+    /**
+     * Check if the cache is empty.
+     */
+    bool empty() const;
+
+    /**
+     * Get the estimated current size in bytes.
+     */
+    size_t size_bytes() const;
+
+    /**
+     * Get the configuration.
+     */
+    const MemoizationConfig& get_config() const { return config_; }
+
+    /**
+     * Get cache statistics.
+     */
+    const MemoizationStats& get_statistics() const { return stats_; }
+
+    /**
+     * Reset statistics (but keep cache contents).
+     */
+    void reset_statistics();
+
+    /**
+     * Enable or disable the cache.
+     * @param enabled Whether to enable caching
+     */
+    void set_enabled(bool enabled);
+
+    /**
+     * Check if caching is enabled.
+     */
+    bool is_enabled() const { return config_.enabled; }
+
+    /**
+     * Get the cache hit rate.
+     * @return Hit rate in range [0.0, 1.0]
+     */
+    float hit_rate() const { return stats_.hit_rate(); }
+
+private:
+    /**
+     * Hash function for prefix vectors.
+     */
+    struct PrefixHasher {
+        size_t operator()(const std::vector<uint8_t>& prefix) const {
+            size_t hash = 0;
+            for (uint8_t byte : prefix) {
+                hash ^= std::hash<uint8_t>{}(byte) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
+            }
+            return hash;
+        }
+    };
+
+    MemoizationConfig config_;
+    mutable MemoizationStats stats_;
+    mutable std::mutex mutex_;
+
+    // LRU list: front = most recently used, back = least recently used
+    std::list<std::vector<uint8_t>> lru_list_;
+    
+    // Map from prefix to (entry, iterator into lru_list)
+    using LRUIterator = std::list<std::vector<uint8_t>>::iterator;
+    std::unordered_map<std::vector<uint8_t>, 
+                       std::pair<PrefixCacheEntry, LRUIterator>,
+                       PrefixHasher> cache_;
+
+    /**
+     * Estimate the size of an entry in bytes.
+     */
+    static size_t estimate_entry_size(const PrefixCacheEntry& entry);
+
+    /**
+     * Evict least recently used entries until under limits.
+     */
+    void evict_if_needed();
+
+    /**
+     * Move a prefix to the front of the LRU list.
+     */
+    void touch(const std::vector<uint8_t>& prefix);
+
+    /**
+     * Convert raw pointer + length to vector for internal use.
+     */
+    static std::vector<uint8_t> to_vector(const uint8_t* data, size_t length);
+};
+
+} // namespace etb
+
+#endif // ETB_MEMOIZATION_HPP

include/etb/path.hpp

@@ -0,0 +1,86 @@
+#ifndef ETB_PATH_HPP
+#define ETB_PATH_HPP
+
+#include "bit_coordinate.hpp"
+#include <vector>
+#include <cstdint>
+#include <stdexcept>
+
+namespace etb {
+
+/**
+ * Represents a forward-only traversal path through bit coordinates.
+ * Enforces the constraint that each subsequent coordinate must have
+ * a strictly greater byte index than the previous one.
+ */
+class Path {
+public:
+    Path() = default;
+    explicit Path(size_t capacity) { coordinates_.reserve(capacity); }
+
+    /**
+     * Add a coordinate to the path.
+     * @param coord The coordinate to add
+     * @return true if added successfully, false if forward-only constraint violated
+     */
+    bool add(const BitCoordinate& coord);
+
+    /**
+     * Validate that the path maintains forward-only traversal.
+     * @return true if all coordinates satisfy byte_index[i] < byte_index[i+1]
+     */
+    bool is_valid() const;
+
+    /**
+     * Get the number of coordinates in the path.
+     */
+    size_t length() const { return coordinates_.size(); }
+
+    /**
+     * Check if the path is empty.
+     */
+    bool empty() const { return coordinates_.empty(); }
+
+    /**
+     * Clear all coordinates from the path.
+     */
+    void clear() { coordinates_.clear(); }
+
+    /**
+     * Reserve capacity for coordinates.
+     */
+    void reserve(size_t capacity) { coordinates_.reserve(capacity); }
+
+    /**
+     * Get coordinate at index.
+     * @throws std::out_of_range if index is invalid
+     */
+    const BitCoordinate& at(size_t index) const { return coordinates_.at(index); }
+
+    /**
+     * Get coordinate at index (no bounds checking).
+     */
+    const BitCoordinate& operator[](size_t index) const { return coordinates_[index]; }
+
+    /**
+     * Get the last coordinate in the path.
+     * @throws std::out_of_range if path is empty
+     */
+    const BitCoordinate& back() const;
+
+    /**
+     * Get read-only access to the underlying coordinates.
+     */
+    const std::vector<BitCoordinate>& coordinates() const { return coordinates_; }
+
+    // Iterator support
+    auto begin() const { return coordinates_.begin(); }
+    auto end() const { return coordinates_.end(); }
+
+private:
+    std::vector<BitCoordinate> coordinates_;
+};
+
+} // namespace etb
+
+#endif // ETB_PATH_HPP

include/etb/path_count.hpp

@@ -0,0 +1,87 @@
+#ifndef ETB_PATH_COUNT_HPP
+#define ETB_PATH_COUNT_HPP
+
+#include <cstdint>
+#include <optional>
+
+namespace etb {
+
+/**
+ * Path count estimation utilities.
+ * 
+ * For n bytes with 8 bits each, the total number of forward-only paths is:
+ * Sum over all path lengths k from 1 to n of: C(n,k) * 8^k
+ * 
+ * This equals (1+8)^n - 1 = 9^n - 1 (by binomial theorem, minus empty path)
+ * 
+ * With bit masking (m allowed bits per byte):
+ * Total paths = (1+m)^n - 1
+ */
+
+/**
+ * Result of path count estimation.
+ */
+struct PathCountResult {
+    uint64_t estimated_count;       // Estimated total path count
+    bool is_exact;                  // True if count is exact (not overflow)
+    bool exceeds_threshold;         // True if count exceeds the given threshold
+    double log_count;               // log10 of the count (for large values)
+};
+
+/**
+ * Calculate the exact number of paths for small inputs.
+ * Returns nullopt if the calculation would overflow uint64_t.
+ * 
+ * @param input_length Number of bytes in input
+ * @param bits_per_byte Number of allowed bit positions per byte (default 8)
+ * @return Exact path count or nullopt on overflow
+ */
+std::optional<uint64_t> exact_path_count(uint32_t input_length, uint8_t bits_per_byte = 8);
+
+/**
+ * Estimate the path count with overflow detection.
+ * 
+ * @param input_length Number of bytes in input
+ * @param bits_per_byte Number of allowed bit positions per byte (default 8)
+ * @param threshold Optional threshold for early bailout check
+ * @return PathCountResult with estimation details
+ */
+PathCountResult estimate_path_count(uint32_t input_length, 
+                                    uint8_t bits_per_byte = 8,
+                                    uint64_t threshold = 0);
+
+/**
+ * Check if path count exceeds a threshold without computing exact count.
+ * Uses logarithmic comparison for efficiency with large values.
+ * 
+ * @param input_length Number of bytes in input
+ * @param bits_per_byte Number of allowed bit positions per byte
+ * @param threshold The threshold to check against
+ * @return true if estimated count exceeds threshold
+ */
+bool path_count_exceeds_threshold(uint32_t input_length,
+                                  uint8_t bits_per_byte,
+                                  uint64_t threshold);
+
+/**
+ * Calculate the log10 of the path count.
+ * Useful for displaying very large counts.
+ * 
+ * @param input_length Number of bytes in input
+ * @param bits_per_byte Number of allowed bit positions per byte
+ * @return log10 of the path count
+ */
+double log10_path_count(uint32_t input_length, uint8_t bits_per_byte = 8);
+
+/**
+ * Count the number of set bits in a byte (popcount).
+ * Used to determine bits_per_byte from a bit mask.
+ * 
+ * @param mask The bit mask
+ * @return Number of set bits
+ */
+uint8_t count_bits_in_mask(uint8_t mask);
+
+} // namespace etb
+
+#endif // ETB_PATH_COUNT_HPP