pybcsv 1.5.10.dev0__tar.gz → 1.5.11.dev0__tar.gz

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybcsv
-Version: 1.5.10.dev0
+Version: 1.5.11.dev0
 Summary: High-performance Python bindings for the BCSV (Binary CSV) library with pandas integration
 Keywords: csv,binary,data-science,compression,performance,pandas,numpy,big-data,data-processing
 Author-Email: Tobias Weber <weber.tobias.md@gmail.com>

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/bitset.hpp

@@ -1606,6 +1606,9 @@ void Bitset<N>::readFrom(const void* src, size_t available) {
     if (available < byteCount()) {
         throw std::out_of_range("Bitset::readFrom: insufficient data");
     }
+    if (byteCount() == 0) {
+        return;  // memcpy with a null src is UB even for n=0
+    }
     std::memcpy(data(), src, byteCount());
     clearUnusedBits();
 }
@@ -1615,6 +1618,9 @@ void Bitset<N>::writeTo(void* dst, size_t capacity) const {
     if (capacity < byteCount()) {
         throw std::out_of_range("Bitset::writeTo: insufficient capacity");
     }
+    if (byteCount() == 0) {
+        return;  // memcpy with a null dst is UB even for n=0
+    }
     // Ensure padding bits in the last word are zero before export
     const_cast<Bitset*>(this)->clearUnusedBits();
     std::memcpy(dst, data(), byteCount());

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_file/file_codec_dispatch.h

@@ -59,6 +59,8 @@ public:
     using FinalizeFn             = void (*)(void* ctx, std::ostream& os, uint64_t totalRows);
     using WriteBufferFn          = ByteBuffer& (*)(void* ctx);
     using ReadRowFn              = std::span<const std::byte> (*)(void* ctx, std::istream& is);
+    using ReadGoodFn             = bool (*)(const void* ctx, std::istream& is);
+    using FinishPacketReadFn     = void (*)(void* ctx, std::istream& is);
     using PacketBoundaryCrossedFn = bool (*)(const void* ctx);
     using ResetFn                = void (*)(void* ctx);
     using SeekToPacketFn         = bool (*)(void* ctx, std::istream& is, std::streamoff offset);
@@ -172,6 +174,25 @@ public:
         return read_row_fn_(ctx_, is);
     }
 
+    /// True while more rows may be readable.  Synchronous codecs report the
+    /// stream state; codecs with an internal I/O thread (batch) answer from
+    /// their own state because the main thread must not poll a stream the
+    /// background thread may be using concurrently.
+    bool readGood(std::istream& is) const {
+        assert(ctx_ && read_good_fn_);
+        return read_good_fn_(ctx_, is);
+    }
+
+    /// Consume + validate the current packet's terminator/checksum without
+    /// touching the next packet.  No-op for codecs that validate the whole
+    /// packet up front (batch) or have no packet framing (stream codecs).
+    /// Throws on checksum mismatch.
+    void finishPacketRead(std::istream& is) {
+        if (ctx_ && finish_packet_read_fn_) {
+            finish_packet_read_fn_(ctx_, is);
+        }
+    }
+
     bool packetBoundaryCrossed() const {
         assert(ctx_ && packet_boundary_crossed_fn_);
         return packet_boundary_crossed_fn_(ctx_);
@@ -218,6 +239,8 @@ private:
     FinalizeFn              finalize_fn_{nullptr};
     WriteBufferFn           write_buffer_fn_{nullptr};
     ReadRowFn               read_row_fn_{nullptr};
+    ReadGoodFn              read_good_fn_{nullptr};
+    FinishPacketReadFn      finish_packet_read_fn_{nullptr};
     PacketBoundaryCrossedFn packet_boundary_crossed_fn_{nullptr};
     ResetFn                 reset_fn_{nullptr};
     SeekToPacketFn          seek_to_packet_fn_{nullptr};
@@ -232,6 +255,8 @@ private:
         finalize_fn_ = nullptr;
         write_buffer_fn_ = nullptr;
         read_row_fn_ = nullptr;
+        read_good_fn_ = nullptr;
+        finish_packet_read_fn_ = nullptr;
         packet_boundary_crossed_fn_ = nullptr;
         reset_fn_ = nullptr;
         seek_to_packet_fn_ = nullptr;
@@ -249,6 +274,8 @@ private:
         finalize_fn_ = other.finalize_fn_;
         write_buffer_fn_ = other.write_buffer_fn_;
         read_row_fn_ = other.read_row_fn_;
+        read_good_fn_ = other.read_good_fn_;
+        finish_packet_read_fn_ = other.finish_packet_read_fn_;
         packet_boundary_crossed_fn_ = other.packet_boundary_crossed_fn_;
         reset_fn_ = other.reset_fn_;
         seek_to_packet_fn_ = other.seek_to_packet_fn_;
@@ -291,6 +318,31 @@ private:
         packet_boundary_crossed_fn_ = [](const void* ctx) -> bool {
             return static_cast<const ConcreteCodec*>(ctx)->packetBoundaryCrossed();
         };
+        // Codecs with an internal I/O thread provide readGood(); synchronous
+        // codecs fall back to plain stream-state polling.
+        if constexpr (requires(const ConcreteCodec& c, std::istream& is) {
+            { c.readGood(is) } -> std::convertible_to<bool>;
+        }) {
+            read_good_fn_ = [](const void* ctx, std::istream& is) -> bool {
+                return static_cast<const ConcreteCodec*>(ctx)->readGood(is);
+            };
+        } else {
+            read_good_fn_ = [](const void* /*ctx*/, std::istream& is) -> bool {
+                return is.good();
+            };
+        }
+        // Synchronous packet codecs provide finishPacketRead(); codecs that
+        // validate whole packets up front (batch) or have no packet framing
+        // (stream) don't need it — the dispatch no-ops for them.
+        if constexpr (requires(ConcreteCodec& c, std::istream& is) {
+            c.finishPacketRead(is);
+        }) {
+            finish_packet_read_fn_ = [](void* ctx, std::istream& is) {
+                static_cast<ConcreteCodec*>(ctx)->finishPacketRead(is);
+            };
+        } else {
+            finish_packet_read_fn_ = nullptr;
+        }
         reset_fn_ = [](void* ctx) {
             static_cast<ConcreteCodec*>(ctx)->reset();
         };

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_file/file_codec_packet001.h

@@ -108,6 +108,9 @@ public:
         }
         FileFooter footer(packet_index_, totalRows);
         footer.write(os);
+        if (!os.good()) {
+            throw std::runtime_error("FileCodecPacket001: footer write failed");
+        }
     }
 
     /// Flush: close the current packet (terminator + checksum), flush the
@@ -193,6 +196,26 @@ public:
         packet_size_ = 0;
     }
 
+    /// Consume the current packet's terminator + checksum and validate.
+    /// Used by direct access after reading a known number of rows: unlike a
+    /// readRow() that crosses the boundary, this STOPS after the checksum —
+    /// it never touches the next packet, so corruption there cannot poison
+    /// reads of this (valid) packet.  Throws on checksum mismatch or a
+    /// malformed packet tail.
+    void finishPacketRead(std::istream& is) {
+        if (!packet_open_) {
+            return;
+        }
+        uint64_t rowLen = 0;
+        vleDecode<uint64_t, true>(is, rowLen, &packet_hash_);
+        if (rowLen != PCKT_TERMINATOR) {
+            throw std::runtime_error(
+                "FileCodecPacket001::finishPacketRead: expected packet terminator");
+        }
+        closePacketRead(is);
+        packet_open_ = false;
+    }
+
     /// Seek to a specific packet by absolute file offset and prepare for reading.
     /// Resets all per-packet state and opens the packet at the given offset.
     /// Returns true if the packet was successfully opened.

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_file/file_codec_packet_lz4_001.h

@@ -155,6 +155,12 @@ public:
         return packet_.seekToPacket(is, offset);
     }
 
+    /// Consume terminator + checksum of the current packet (framing lives in
+    /// the inner packet codec — see FileCodecPacket001::finishPacketRead).
+    void finishPacketRead(std::istream& is) {
+        packet_.finishPacketRead(is);
+    }
+
 private:
     ByteBuffer write_buffer_;   // Owned write buffer for RowCodec serialization
     FileCodecPacket001 packet_;         // Handles framing, checksums, packet lifecycle

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_file/file_codec_packet_lz4_batch001.h

@@ -37,7 +37,18 @@
  *   BLE(PCKT_TERMINATOR)
  *
  * Exceptions that occur on the background thread are captured via
- * std::exception_ptr and re-thrown on the next main-thread call.
+ * std::exception_ptr (under mutex_) and re-thrown on the main thread at the
+ * next synchronization point: packet boundary, flushPacket(), or finalize().
+ *
+ * Threading contract (main thread ↔ one internal BG thread):
+ *  - While a BG task is in flight, the BG thread is the only owner of the
+ *    stream.  The main thread must not touch the stream — not even to poll
+ *    rdstate(); it queries readGood() instead, which answers from
+ *    main-thread-owned state (packet_open_).
+ *  - The BG thread leaves the stream in a defined state on every task exit
+ *    (clears eof/fail bits it caused; EOF is reported exclusively via
+ *    bg_has_next_packet_).
+ *  - bg_exception_ is written and read only under mutex_.
  */
 
 #include "file_codec_concept.h"
@@ -64,6 +75,7 @@
 #include <stop_token>
 #include <string>
 #include <thread>
+#include <utility>
 
 namespace bcsv {
 
@@ -127,9 +139,9 @@ public:
 
     /// Called before each writeRow().  Returns true if a packet boundary
     /// was crossed (caller resets RowCodec).
+    /// Note: BG failures surface at the next packet boundary, flushPacket()
+    /// or finalize() — never on the per-row fast path (no locking here).
     bool beginWrite(std::ostream& /*os*/, uint64_t rowCnt) {
-        rethrowBgException();
-
         if (raw_active_->size() >= packet_size_limit_) {
             // Close the current packet payload.
             vleEncode<uint64_t, true>(static_cast<uint64_t>(PCKT_TERMINATOR), *raw_active_);
@@ -169,6 +181,9 @@ public:
     }
 
     /// Finalize: flush remaining data, shut down BG thread, write footer.
+    /// Any pending BG failure is rethrown unconditionally BEFORE the footer
+    /// is written — a file with a corrupt/missing packet must never get a
+    /// clean footer.
     void finalize(std::ostream& /*os*/, uint64_t totalRows) {
         if (!raw_active_->empty()) {
             // Close the last packet payload.
@@ -184,16 +199,18 @@ public:
                 bg_task_ = BgTask::COMPRESS_WRITE;
             }
             cv_.notify_one();
-
-            waitForBgIdle();
-            rethrowBgException();
         }
 
-        shutdownBgThread();
+        shutdownBgThread();      // waits for any in-flight task, then joins
+        rethrowBgException();    // unconditional — surfaces failures from the last packet
 
         // Write file footer on main thread (BG is stopped).
         FileFooter footer(packet_index_, totalRows);
         footer.write(*os_ptr_);
+        if (!os_ptr_->good()) {
+            throw std::runtime_error(
+                "FileCodecPacketLZ4Batch001: footer write failed");
+        }
     }
 
     ByteBuffer& writeBuffer() { return write_buffer_; }
@@ -233,8 +250,10 @@ public:
 
     // ── Read lifecycle ──────────────────────────────────────────────────
 
+    /// Note: BG failures (e.g. a corrupt pre-read packet) surface at the
+    /// packet boundary inside decodeNextRow() — remaining rows of the
+    /// current (validated) packet are served first; no locking per row.
     std::span<const std::byte> readRow(std::istream& /*is*/) {
-        rethrowBgException();
         packet_boundary_crossed_ = false;
 
         if (!packet_open_) {
@@ -246,6 +265,14 @@ public:
 
     // ── Boundary / state signals ────────────────────────────────────────
 
+    /// Reader liveness query — replaces direct stream polling.  The BG
+    /// thread may be touching the stream concurrently, so the main thread
+    /// must not inspect stream state; packet_open_ is main-thread-owned
+    /// and reflects whether more rows may be available.
+    bool readGood(std::istream& /*is*/) const noexcept {
+        return packet_open_;
+    }
+
     bool packetBoundaryCrossed() const noexcept {
         return packet_boundary_crossed_;
     }
@@ -290,6 +317,8 @@ private:
     void startBgThread() {
         if (bg_thread_.joinable()) return;  // Already running
 
+        // No BG thread exists here, so plain writes are race-free; the
+        // jthread constructor below provides the happens-before edge.
         bg_task_ = BgTask::IDLE;
         bg_exception_ = nullptr;
         bg_thread_ = std::jthread([this](std::stop_token stoken) { bgLoop(stoken); });
@@ -317,7 +346,19 @@ private:
         // fires cv_.notify_all() when request_stop() is called, ensuring
         // the BG thread exits its idle wait without a manual CV notification
         // from the main thread.
-        std::stop_callback wake_on_stop(stoken, [this] { cv_.notify_all(); });
+        //
+        // The lock_guard is essential: request_stop() sets the stop flag
+        // OUTSIDE mutex_, so an unlocked notify could land in the window
+        // where this thread holds the mutex between a (pre-stop) predicate
+        // check and going to sleep — a lost wakeup that leaves the CV wait
+        // sleeping and shutdownBgThread()'s join() blocked forever.  Taking
+        // the mutex serializes the notify against the predicate check.
+        // (Reproduced readily by open()+close() cycles under load: the
+        // shutdown races the just-started thread's first predicate check.)
+        std::stop_callback wake_on_stop(stoken, [this] {
+            std::lock_guard<std::mutex> lk(mutex_);
+            cv_.notify_all();
+        });
 
         while (!stoken.stop_requested()) {
             std::unique_lock<std::mutex> lk(mutex_);
@@ -328,6 +369,7 @@ private:
             BgTask task = bg_task_;
             lk.unlock();
 
+            std::exception_ptr ex;
             try {
                 if (task == BgTask::COMPRESS_WRITE) {
                     bgCompressAndWrite();
@@ -335,11 +377,14 @@ private:
                     bgReadAndDecompress();
                 }
             } catch (...) {
-                bg_exception_ = std::current_exception();
+                ex = std::current_exception();
             }
 
             {
                 std::lock_guard<std::mutex> lk2(mutex_);
+                if (ex) {
+                    bg_exception_ = ex;  // published under mutex_ only
+                }
                 bg_task_ = BgTask::IDLE;
             }
             cv_.notify_one();
@@ -351,10 +396,16 @@ private:
         cv_.wait(lk, [this] { return bg_task_ == BgTask::IDLE; });
     }
 
+    /// Take and rethrow a captured BG exception, if any.  Safe to call at
+    /// any time (slot is accessed under mutex_), but only used at packet
+    /// boundaries / flush / finalize — never on the per-row fast path.
     void rethrowBgException() {
-        if (bg_exception_) {
-            std::exception_ptr ex = bg_exception_;
-            bg_exception_ = nullptr;
+        std::exception_ptr ex;
+        {
+            std::lock_guard<std::mutex> lk(mutex_);
+            ex = std::exchange(bg_exception_, nullptr);
+        }
+        if (ex) {
             std::rethrow_exception(ex);
         }
     }
@@ -425,6 +476,12 @@ private:
                 is.seekg(pos);
                 return false;   // Footer reached
             } else if (is.eof()) {
+                // Raw EOF (footer-less / crashed file): restore a defined
+                // stream state.  End-of-data is reported to the main thread
+                // exclusively via the return value (bg_has_next_packet_) —
+                // never via stream flags, which the main thread must not
+                // poll while this may run on the BG thread.
+                is.clear();
                 return false;
             } else {
                 is.clear();
@@ -448,9 +505,19 @@ private:
                 "FileCodecPacketLZ4Batch001: failed to read compressed_size");
         }
 
-        if (uncompressed_size > MAX_PACKET_SIZE || compressed_size > MAX_PACKET_SIZE) {
+        // Bound declared sizes by what this file's header allows, not by the
+        // absolute 1 GiB format limit: a packet payload can exceed the
+        // header packet size by at most one row (+ VLE framing + terminator).
+        // This stops tiny corrupt/hostile files from triggering huge
+        // allocations before checksum validation gets a chance to reject them.
+        const uint64_t max_uncompressed =
+            static_cast<uint64_t>(packet_size_limit_) + MAX_ROW_LENGTH + 16;
+        // LZ4 worst-case bound (LZ4_COMPRESSBOUND formula) inlined in 64-bit:
+        // the macro/API take int and would overflow for large packet sizes.
+        const uint64_t max_compressed = max_uncompressed + max_uncompressed / 255 + 16;
+        if (uncompressed_size > max_uncompressed || compressed_size > max_compressed) {
             throw std::runtime_error(
-                "FileCodecPacketLZ4Batch001: packet size exceeds MAX_PACKET_SIZE");
+                "FileCodecPacketLZ4Batch001: declared packet size exceeds header packet size limit");
         }
 
         // 3. Read compressed block.
@@ -599,9 +666,16 @@ private:
     // mutex + cv_.notify) and read by main AFTER waitForBgIdle():
     //   bg_has_next_packet_ — whether read_next_ contains valid data
     //   packet_index_       — BG appends entries; main reads after join()
-    //   bg_exception_       — captured exception; main reads after wait
     //   compressed_buf_     — BG writes here; main never reads it directly
     //
+    // bg_exception_ is written and read ONLY under mutex_ (BG publishes in
+    // the same critical section as the IDLE transition; main takes it via
+    // rethrowBgException() at packet boundaries / flush / finalize).
+    //
+    // The stream (is_ptr_/os_ptr_) is owned by the BG thread while a task
+    // is in flight.  The main thread must not touch it then — not even
+    // rdstate(); Reader queries readGood() instead (main-thread state).
+    //
     // All other state is exclusively owned by one thread (documented below).
 
     // Main-thread only: RowCodec serialization buffer.
@@ -650,7 +724,7 @@ private:
     std::mutex               mutex_;       ///< Protects bg_task_ and provides happens-before edges.
     std::condition_variable  cv_;          ///< Signals bg_task_ transitions.
     BgTask                   bg_task_{BgTask::IDLE};
-    std::exception_ptr       bg_exception_;///< BG→main via sync protocol.
+    std::exception_ptr       bg_exception_;///< Guarded by mutex_ (see protocol above).
 };
 
 static_assert(FileCodecConcept<FileCodecPacketLZ4Batch001>,

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_row/row_codec_delta002.h

@@ -69,6 +69,27 @@
 
 namespace bcsv {
 
+/// Cold path of delta002ValidateLengthCode — kept out of line so the string
+/// construction does not bloat the decode hot loop.
+[[noreturn]] inline void delta002ThrowInvalidLengthCode(size_t deltaBytes,
+                                                        size_t typeSize) {
+    throw std::runtime_error(
+        "RowCodecDelta002::deserialize() failed! Invalid delta length code ("
+        + std::to_string(deltaBytes) + " bytes for a "
+        + std::to_string(typeSize) + "-byte column)");
+}
+
+/// Reject a delta header code that implies more payload bytes than the
+/// column type holds.  Reachable only from corrupt/hostile input (the
+/// encoder clamps to sizeof(T)); without this check decodeDelta() would
+/// shift past the 64-bit width (UB) and desync all following column
+/// offsets.  Shared by the dynamic and static Delta002 decoders.
+inline void delta002ValidateLengthCode(size_t deltaBytes, size_t typeSize) {
+    if (deltaBytes > typeSize) [[unlikely]] {
+        delta002ThrowInvalidLengthCode(deltaBytes, typeSize);
+    }
+}
+
 // ────────────────────────────────────────────────────────────────────────────
 // RowCodecDelta002 for dynamic Layout
 // ────────────────────────────────────────────────────────────────────────────
@@ -425,7 +446,10 @@ private:
 
     static uint64_t decodeDelta(const std::byte* src, size_t byteCount) {
         uint64_t result = 0;
-        for (size_t i = 0; i < byteCount; ++i)
+        // Clamped loop bound: total for all inputs + full unrolling.
+        // See the dynamic-layout decodeDelta in row_codec_delta002.hpp.
+        const size_t n = byteCount <= sizeof(uint64_t) ? byteCount : sizeof(uint64_t);
+        for (size_t i = 0; i < n; ++i)
             result |= static_cast<uint64_t>(static_cast<uint8_t>(src[i])) << (i * 8);
         return result;
     }

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_row/row_codec_delta002.hpp

@@ -24,6 +24,7 @@
 #include <algorithm>
 #include <bit>
 #include <cassert>
+#include <cmath>
 #include <cstring>
 #include <type_traits>
 
@@ -50,7 +51,14 @@ size_t RowCodecDelta002<LayoutType>::encodeDelta(std::byte* dst, uint64_t value,
 template<typename LayoutType>
 uint64_t RowCodecDelta002<LayoutType>::decodeDelta(const std::byte* src, size_t byteCount) {
     uint64_t result = 0;
-    for (size_t i = 0; i < byteCount; ++i)
+    // Clamp the loop bound: makes the function total (no shift past the
+    // 64-bit width for any input) and, importantly, gives the compiler a
+    // provable max trip count — the byte-assembly loop unrolls fully,
+    // which measures FASTER than the unclamped loop (see docs/archive/
+    // B2_VALIDATION_COST_INVESTIGATION.md).  For valid inputs (encoder
+    // clamps to sizeof(T) <= 8) the clamp is an identity.
+    const size_t n = byteCount <= sizeof(uint64_t) ? byteCount : sizeof(uint64_t);
+    for (size_t i = 0; i < n; ++i)
         result |= static_cast<uint64_t>(static_cast<uint8_t>(src[i])) << (i * 8);
     return result;
 }
@@ -155,6 +163,13 @@ bool RowCodecDelta002<LayoutType>::checkFloatFoC(const std::byte* curr, const st
     std::memcpy(&p, prev, sizeof(T));
     std::memcpy(&g, grad, sizeof(T));
     T predicted = p + g;
+    // Never emit FoC for a NaN prediction: the decoder reconstructs the
+    // value as prev + grad, and NaN *payload* propagation through
+    // arithmetic is implementation-defined — the XOR-delta path is
+    // bit-exact on every platform.  (Encoder-only guard; wire unchanged.)
+    if (std::isnan(predicted)) {
+        return false;
+    }
     return std::memcmp(&c, &predicted, sizeof(T)) == 0;
 }
 
@@ -453,6 +468,7 @@ void RowCodecDelta002<LayoutType>::deserialize(
 
             // ── code ≥ 2: delta with (code-1) bytes ─────────────────────
             const size_t deltaBytes = code - 1;
+            delta002ValidateLengthCode(deltaBytes, sizeof(T));
 
             if (dataOff + deltaBytes > buffer.size())
                 throw std::runtime_error(
@@ -627,6 +643,10 @@ bool RowCodecDelta002<LayoutStatic<ColumnTypes...>>::checkFloatFoC(
     const T& curr, const T& prev, const T& grad)
 {
     T predicted = prev + grad;
+    // See the dynamic-layout checkFloatFoC: no FoC for NaN predictions.
+    if (std::isnan(predicted)) {
+        return false;
+    }
     return std::memcmp(&curr, &predicted, sizeof(T)) == 0;
 }
 
@@ -694,7 +714,9 @@ RowCodecDelta002<LayoutStatic<ColumnTypes...>>::serialize(
                 T& grad = std::get<ColIdx>(grad_data_);
 
                 // ── ZoH check ────────────────────────────────────────────
-                if (curr == prev) {
+                // bitEqual: NaN holds (same bits) and -0.0 vs +0.0 counts
+                // as a change (matches the dynamic layout's memcmp check).
+                if (bitEqual(curr, prev)) {
                     row_header_.encode(ROW_HEADER_BIT_INDEX[ColIdx], HB, 0);
                     grad = T{};
                     return;
@@ -846,6 +868,7 @@ void RowCodecDelta002<LayoutStatic<ColumnTypes...>>::deserialize(
 
                 // ── code ≥ 2: delta with (code-1) bytes ─────────────────
                 const size_t deltaBytes = code - 1;
+                delta002ValidateLengthCode(deltaBytes, sizeof(T));
 
                 if (dataOff + deltaBytes > buffer.size())
                     throw std::runtime_error(

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_row/row_codec_flat001.hpp

@@ -75,10 +75,13 @@ inline std::span<std::byte> RowCodecFlat001<LayoutType>::serialize(
     const uint32_t*   offsets = layout_->columnOffsets().data();
 
     // ── Pre-scan: sum string payload sizes for a single buffer.resize() ──
+    // Clamp to MAX_STRING_LENGTH — the write loop below truncates to that
+    // limit, and the buffer size must match the bytes actually written
+    // (an unclamped size would leave uninitialized bytes in the row span).
     size_t strg_payload = 0;
     for (size_t i = 0; i < count; ++i) {
         if (types[i] == ColumnType::STRING) {
-            strg_payload += row.strg_[offsets[i]].size();
+            strg_payload += std::min(row.strg_[offsets[i]].size(), MAX_STRING_LENGTH);
         }
     }
 
@@ -195,12 +198,14 @@ inline std::span<std::byte> RowCodecFlat001<LayoutStatic<ColumnTypes...>>::seria
 
     const size_t offRow = buffer.size();
 
-    // Compute total string payload via fold expression
+    // Compute total string payload via fold expression.  Clamped to
+    // MAX_STRING_LENGTH to match the truncating write in serializeElements
+    // (see the dynamic-layout pre-scan above).
     size_t strg_payload = 0;
     [&]<size_t... I>(std::index_sequence<I...>) {
         (([&] {
             if constexpr (std::is_same_v<typename RowType::template column_type<I>, std::string>) {
-                strg_payload += std::get<I>(row.data_).size();
+                strg_payload += std::min(std::get<I>(row.data_).size(), MAX_STRING_LENGTH);
             }
         }()), ...);
     }(std::index_sequence_for<ColumnTypes...>{});

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/codec_row/row_codec_zoh001.hpp

@@ -446,7 +446,10 @@ RowCodecZoH001<LayoutStatic<ColumnTypes...>>::serialize(
                     std::get<I>(prev_data_) = std::get<I>(row.data_);
                 }
             } else {
-                if (std::get<I>(row.data_) != std::get<I>(prev_data_)) {
+                // bitEqual: NaN holds (same bits) and -0.0 vs +0.0 counts as
+                // a change — IEEE operator== would silently turn -0.0 into
+                // the held +0.0 on the decoder side.
+                if (!bitEqual(std::get<I>(row.data_), std::get<I>(prev_data_))) {
                     row_header_.set(ROW_HEADER_BIT_INDEX[I], true);
                     hasAnyChange = true;
                     std::get<I>(prev_data_) = std::get<I>(row.data_);

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/definitions.h

@@ -41,7 +41,35 @@
 #endif
 
 namespace bcsv {
-    
+
+    // The wire format is written by memcpy of native integers and documented
+    // as little-endian.  Until explicit byte-order conversion exists, refuse
+    // to compile on big-endian targets rather than silently producing files
+    // no other platform can read.
+    static_assert(std::endian::native == std::endian::little,
+                  "BCSV wire format requires a little-endian target");
+
+    /// Value equality with bit-exact semantics for floating-point types:
+    /// NaN equals NaN (same bit pattern) and -0.0 does NOT equal +0.0.
+    /// Codec change detection (ZoH/Delta) must use this instead of
+    /// operator== so signed zeros round-trip exactly and repeated NaNs
+    /// hold — IEEE operator== breaks both (NaN != NaN; -0.0 == +0.0).
+    /// Non-floating types fall back to operator==.
+    /// BCSV_ALWAYS_INLINE: without it, the extra call in the templated
+    /// serialize fold-expressions perturbs GCC's TU-wide inlining budget and
+    /// measurably degrades unrelated (decode) hot loops — see
+    /// docs/archive/B2_VALIDATION_COST_INVESTIGATION.md for the same effect class.
+    template<typename T>
+    [[nodiscard]] BCSV_ALWAYS_INLINE bool bitEqual(const T& a, const T& b) {
+        if constexpr (std::is_same_v<T, float>) {
+            return std::bit_cast<uint32_t>(a) == std::bit_cast<uint32_t>(b);
+        } else if constexpr (std::is_same_v<T, double>) {
+            return std::bit_cast<uint64_t>(a) == std::bit_cast<uint64_t>(b);
+        } else {
+            return a == b;
+        }
+    }
+
     // Configuration
     constexpr bool RANGE_CHECKING = true;
     constexpr bool STRING_OVERFLOW_THROWS = true;  // true = throw std::length_error on string > MAX_STRING_LENGTH, false = silently truncate (legacy behavior)
@@ -80,7 +108,8 @@ namespace bcsv {
     constexpr size_t MAX_COLUMN_COUNT  = std::numeric_limits<uint16_t>::max();  // Maximum number of columns
     constexpr size_t MAX_COLUMN_LENGTH = std::numeric_limits<uint16_t>::max();  // Maximum width of column content
     constexpr size_t MAX_STRING_LENGTH = std::numeric_limits<uint16_t>::max(); // Maximum length of string data (wire format uses uint16_t lengths)
-    constexpr size_t MAX_ROW_LENGTH    = (1ULL << 24) - 2 ;                     // ~16 MB maximum row size, 4b BLE encoding (2 bits for length), reserve 0xFFFF for terminator.
+    constexpr size_t MAX_ROW_LENGTH    = (1ULL << 24) - 2 ;                     // ~16 MB maximum row size, 4b BLE encoding (2 bits for length); safely below PCKT_TERMINATOR (0x3FFFFFFF) so a row length can never collide with the terminator marker.
+    constexpr size_t MAX_HEADER_NAME_BYTES = 16 * 1024 * 1024;              // Cumulative cap on column-name bytes in a file header (hostile-input guard)
     constexpr size_t MIN_PACKET_SIZE         = 64 * 1024;                       // 64KB minimum packet size
     constexpr size_t DEFAULT_PACKET_SIZE_KB  = 8192;                            // 8MB default packet size (in KB, for Writer::open blockSizeKB param)
     constexpr size_t MAX_PACKET_SIZE         = 1024 * 1024 * 1024;              // 1GB maximum packet size

{pybcsv-1.5.10.dev0 → pybcsv-1.5.11.dev0}/include/bcsv/file_footer.h

@@ -246,6 +246,18 @@ namespace bcsv {
                 return false;
             }
 
+            // Validate start_offset before any arithmetic on it: it must
+            // cover at least magic + ConstSection (or `indexSize` below
+            // underflows size_t), and it cannot reach past the start of the
+            // file (this also bounds the index allocation by the file size).
+            stream.seekg(0, std::ios::end);
+            const std::streamoff fileSize = stream.tellg();
+            if (const_section_.start_offset < sizeof(ConstSection) + 4 ||
+                static_cast<std::streamoff>(const_section_.start_offset) > fileSize) {
+                stream.seekg(originalPos);
+                return false;
+            }
+
             // Seek to index start
             stream.seekg(-static_cast<int64_t>(const_section_.start_offset), std::ios::end);