logosdb 0.2.0__tar.gz → 0.2.2__tar.gz

{logosdb-0.2.0 → logosdb-0.2.2}/CHANGELOG

@@ -1,6 +1,32 @@
 logosdb change log
 ==================
 
+0.2.2  (2026-04-20)
+-------------------
+
+  * Introduced Write-Ahead Log (WAL) for atomic Put operations. Closes #2.
+    - New `src/wal.{h,cpp}` implementing `WriteAheadLog` class with binary
+      append-only format, state tracking (PENDING/COMMITTED), and replay.
+    - `logosdb_put` now writes to WAL first (durability point), then modifies
+      vector storage, metadata, and HNSW index. On success, WAL entry is marked
+      committed; on crash, pending entries are replayed on next open.
+    - `logosdb_open` replays any pending WAL entries before serving requests,
+      ensuring consistency across all three stores after partial failures.
+    - New `wal.log` file in database directory; backward compatible with
+      existing databases (WAL is empty on first open).
+    - New test `test_wal_crash_recovery` validates crash recovery behavior.
+
+0.2.1  (2026-04-20)
+-------------------
+
+  * Replaced hand-rolled JSON parser with nlohmann/json (v3.11.3, single-header,
+    vendored in third_party/nlohmann/). This fixes multiple parsing edge cases:
+    unicode escape sequences (\uXXXX), key ordering independence, extra
+    whitespace tolerance, and proper escape handling. Closes #5.
+    Added regression tests for unicode, empty strings, complex backslash
+    escapes, and key-order variations. All 130 C++ tests pass; total assertions
+    up from 115 to 130.
+
 0.2.0  (2026-04-17)
 -------------------
 

{logosdb-0.2.0 → logosdb-0.2.2}/CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.15)
-project(logosdb VERSION 0.2.0 LANGUAGES CXX)
+project(logosdb VERSION 0.2.2 LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -8,22 +8,26 @@ option(LOGOSDB_BUILD_TOOLS  "Build CLI and benchmark tools" ON)
 option(LOGOSDB_BUILD_TESTS  "Build unit tests"              ON)
 option(LOGOSDB_BUILD_PYTHON "Build Python bindings (pybind11)" OFF)
 
-# hnswlib is header-only
+# hnswlib and nlohmann/json are header-only
 add_library(hnswlib INTERFACE)
 target_include_directories(hnswlib INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
 
+add_library(nlohmann_json INTERFACE)
+target_include_directories(nlohmann_json INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
+
 # Core library
 add_library(logosdb
     src/logosdb.cpp
     src/storage.cpp
     src/metadata.cpp
     src/hnsw_index.cpp
+    src/wal.cpp
 )
 target_include_directories(logosdb PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}/include
     ${CMAKE_CURRENT_SOURCE_DIR}/src
 )
-target_link_libraries(logosdb PRIVATE hnswlib)
+target_link_libraries(logosdb PRIVATE hnswlib nlohmann_json)
 
 # Position-independent code is required when the static library is linked
 # into a shared object (the Python extension is a .so).

{logosdb-0.2.0 → logosdb-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: logosdb
-Version: 0.2.0
+Version: 0.2.2
 Summary: Fast semantic vector database (HNSW + mmap) with Python bindings
 Keywords: vector-database,hnsw,embeddings,semantic-search,nearest-neighbor
 Author-Email: Jose <jose-compu@users.noreply.github.com>
@@ -233,7 +233,7 @@ Here is a performance report from the included `logosdb-bench` program. The resu
 
 We use databases with 1K, 10K, and 100K vectors. Each vector has 2048 dimensions (matching typical LLM embedding sizes). Vectors are L2-normalized random unit vectors.
 
-    LogosDB:    version 0.2.0
+    LogosDB:    version 0.2.2
     CPU:        Apple M-series (ARM64)
     Dim:        2048
     HNSW M:     16, ef_construction: 200, ef_search: 50

{logosdb-0.2.0 → logosdb-0.2.2}/README.md

@@ -185,7 +185,7 @@ Here is a performance report from the included `logosdb-bench` program. The resu
 
 We use databases with 1K, 10K, and 100K vectors. Each vector has 2048 dimensions (matching typical LLM embedding sizes). Vectors are L2-normalized random unit vectors.
 
-    LogosDB:    version 0.2.0
+    LogosDB:    version 0.2.2
     CPU:        Apple M-series (ARM64)
     Dim:        2048
     HNSW M:     16, ef_construction: 200, ef_search: 50

{logosdb-0.2.0 → logosdb-0.2.2}/include/logosdb/logosdb.h

@@ -6,8 +6,8 @@
 
 #define LOGOSDB_VERSION_MAJOR 0
 #define LOGOSDB_VERSION_MINOR 2
-#define LOGOSDB_VERSION_PATCH 0
-#define LOGOSDB_VERSION_STRING "0.2.0"
+#define LOGOSDB_VERSION_PATCH 2
+#define LOGOSDB_VERSION_STRING "0.2.2"
 
 #ifdef __cplusplus
 extern "C" {

{logosdb-0.2.0 → logosdb-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "logosdb"
-version = "0.2.0"
+version = "0.2.2"
 description = "Fast semantic vector database (HNSW + mmap) with Python bindings"
 readme = "README.md"
 license = { file = "LICENSE" }

{logosdb-0.2.0 → logosdb-0.2.2}/src/logosdb.cpp

@@ -2,10 +2,12 @@
 #include "storage.h"
 #include "metadata.h"
 #include "hnsw_index.h"
+#include "wal.h"
 
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
+#include <functional>
 #include <mutex>
 #include <string>
 #include <vector>
@@ -15,11 +17,12 @@ using namespace logosdb::internal;
 /* ── Internal DB struct ────────────────────────────────────────────── */
 
 struct logosdb_t {
-    VectorStorage  vectors;
-    MetadataStore  meta;
-    HnswIndex      index;
-    std::mutex     mu;
-    int            dim = 0;
+    VectorStorage   vectors;
+    MetadataStore   meta;
+    HnswIndex       index;
+    WriteAheadLog   wal;
+    std::mutex      mu;
+    int             dim = 0;
 };
 
 struct logosdb_options_t {
@@ -80,6 +83,7 @@ logosdb_t * logosdb_open(const char * path, const logosdb_options_t * opts,
     std::string vec_path  = std::string(path) + "/vectors.bin";
     std::string meta_path = std::string(path) + "/meta.jsonl";
     std::string idx_path  = std::string(path) + "/hnsw.idx";
+    std::string wal_path  = std::string(path) + "/wal.log";
 
     if (!db->vectors.open(vec_path, opts->dim, err)) {
         set_err(errptr, err);
@@ -105,6 +109,48 @@ logosdb_t * logosdb_open(const char * path, const logosdb_options_t * opts,
         return nullptr;
     }
 
+    // Open WAL and replay any pending entries for atomic recovery.
+    if (!db->wal.open(wal_path, err)) {
+        set_err(errptr, err);
+        delete db;
+        return nullptr;
+    }
+
+    // Replay pending WAL entries to ensure consistency.
+    int replayed = db->wal.replay_pending(
+        [&db](const WALEntry & entry, std::string & replay_err) -> bool {
+            // Validate: expected_id should match current row count
+            if (entry.expected_id != db->vectors.n_rows()) {
+                replay_err = "wal replay: expected_id mismatch (" +
+                             std::to_string(entry.expected_id) + " vs " +
+                             std::to_string(db->vectors.n_rows()) + ")";
+                return false;
+            }
+
+            // Replay vector
+            uint64_t vid = db->vectors.append(entry.vector.data(), (int)entry.dim, replay_err);
+            if (vid == UINT64_MAX) return false;
+
+            // Replay metadata
+            uint64_t mid = db->meta.append(entry.text.c_str(), entry.timestamp.c_str(), replay_err);
+            if (mid == UINT64_MAX) return false;
+
+            // Replay index
+            if (!db->index.add(vid, entry.vector.data(), replay_err)) {
+                return false;
+            }
+
+            return true;
+        },
+        err
+    );
+
+    if (replayed < 0) {
+        set_err(errptr, "wal replay: " + err);
+        delete db;
+        return nullptr;
+    }
+
     // Backfill index if vector storage has more rows than the index (e.g. crash recovery).
     size_t n_vec = db->vectors.n_rows();
     size_t n_idx = db->index.count();
@@ -144,9 +190,11 @@ logosdb_t * logosdb_open(const char * path, const logosdb_options_t * opts,
 void logosdb_close(logosdb_t * db) {
     if (!db) return;
     std::string err;
+    db->wal.sync(err);   // Ensure WAL is durable before closing other stores
     db->index.save(err);
     db->vectors.sync(err);
     db->meta.sync(err);
+    db->wal.close();
     delete db;
 }
 
@@ -169,22 +217,38 @@ uint64_t logosdb_put(logosdb_t * db,
     std::lock_guard<std::mutex> lock(db->mu);
     std::string err;
 
-    // NOTE: these three writes are not atomic. On partial failure the stores
-    // may diverge (e.g. vectors written but metadata missing). The HNSW index
-    // is backfilled from the vector store on next open, but metadata gaps are
-    // not currently recoverable. A WAL would fix this; acceptable for now
-    // given the single-process embedded use case.
+    // Compute expected row id before writing
+    uint64_t expected_id = db->vectors.n_rows();
+
+    // Step 1: Write WAL entry (durability point)
+    int64_t wal_offset = db->wal.append_pending(embedding, dim, text, timestamp, expected_id, err);
+    if (wal_offset < 0) { set_err(errptr, err); return UINT64_MAX; }
+
+    // Step 2: Write to vector storage
     uint64_t vid = db->vectors.append(embedding, dim, err);
     if (vid == UINT64_MAX) { set_err(errptr, err); return UINT64_MAX; }
 
+    // Step 3: Write to metadata storage
     uint64_t mid = db->meta.append(text, timestamp, err);
-    if (mid == UINT64_MAX) { set_err(errptr, err); return UINT64_MAX; }
+    if (mid == UINT64_MAX) {
+        // Metadata write failed - entry remains in WAL for replay on recovery
+        set_err(errptr, err);
+        return UINT64_MAX;
+    }
 
+    // Step 4: Write to HNSW index
     if (!db->index.add(vid, embedding, err)) {
+        // Index write failed - entry remains in WAL for replay on recovery
         set_err(errptr, err);
         return UINT64_MAX;
     }
 
+    // Step 5: Mark WAL entry as committed
+    if (!db->wal.mark_committed(wal_offset, err)) {
+        // Non-fatal: entry will be replayed on next open if needed
+        // Log but don't fail the operation
+    }
+
     return vid;
 }
 

{logosdb-0.2.0 → logosdb-0.2.2}/src/metadata.cpp

@@ -1,65 +1,17 @@
 #include "metadata.h"
 
+#include <nlohmann/json.hpp>
+
 #include <cerrno>
 #include <cstring>
 #include <fstream>
-#include <sstream>
 #include <fcntl.h>
 #include <unistd.h>
 
 namespace logosdb {
 namespace internal {
 
-// Minimal JSON helpers — no external dependency.
-
-static std::string json_escape(const std::string & s) {
-    std::string out;
-    out.reserve(s.size() + 8);
-    for (char c : s) {
-        switch (c) {
-            case '"':  out += "\\\""; break;
-            case '\\': out += "\\\\"; break;
-            case '\n': out += "\\n";  break;
-            case '\r': out += "\\r";  break;
-            case '\t': out += "\\t";  break;
-            default:   out += c;
-        }
-    }
-    return out;
-}
-
-static std::string json_unescape(const std::string & s) {
-    std::string out;
-    out.reserve(s.size());
-    for (size_t i = 0; i < s.size(); ++i) {
-        if (s[i] == '\\' && i + 1 < s.size()) {
-            switch (s[i + 1]) {
-                case '"':  out += '"';  ++i; break;
-                case '\\': out += '\\'; ++i; break;
-                case 'n':  out += '\n'; ++i; break;
-                case 'r':  out += '\r'; ++i; break;
-                case 't':  out += '\t'; ++i; break;
-                default:   out += s[i];
-            }
-        } else {
-            out += s[i];
-        }
-    }
-    return out;
-}
-
-static std::string extract_field(const std::string & line, const std::string & key) {
-    std::string needle = "\"" + key + "\":\"";
-    auto pos = line.find(needle);
-    if (pos == std::string::npos) return "";
-    pos += needle.size();
-    auto end = line.find('"', pos);
-    while (end != std::string::npos && end > 0 && line[end - 1] == '\\') {
-        end = line.find('"', end + 1);
-    }
-    if (end == std::string::npos) return "";
-    return json_unescape(line.substr(pos, end - pos));
-}
+using json = nlohmann::json;
 
 MetadataStore::~MetadataStore() { close(); }
 
@@ -78,29 +30,34 @@ bool MetadataStore::open(const std::string & path, std::string & err) {
         std::string line;
         while (std::getline(in, line)) {
             if (line.empty()) continue;
+
+            // Try to parse as JSON
+            json j;
+            try {
+                j = json::parse(line);
+            } catch (const json::exception & e) {
+                // Invalid JSON line - skip but don't fail
+                continue;
+            }
+
             // Tombstone record: {"op":"del","id":N}
-            if (line.find("\"op\":\"del\"") != std::string::npos) {
-                auto key = std::string("\"id\":");
-                auto pos = line.find(key);
-                if (pos == std::string::npos) continue;
-                pos += key.size();
-                uint64_t id = 0;
-                bool got_digit = false;
-                while (pos < line.size() && (line[pos] == ' ' || line[pos] == '\t')) ++pos;
-                while (pos < line.size() && line[pos] >= '0' && line[pos] <= '9') {
-                    id = id * 10 + (uint64_t)(line[pos] - '0');
-                    got_digit = true;
-                    ++pos;
-                }
-                if (got_digit && id < rows_.size() && !rows_[id].deleted) {
+            if (j.contains("op") && j["op"] == "del" && j.contains("id")) {
+                uint64_t id = j["id"].get<uint64_t>();
+                if (id < rows_.size() && !rows_[id].deleted) {
                     rows_[id].deleted = true;
                     ++num_deleted_;
                 }
                 continue;
             }
+
+            // Data row: {"text":"...","ts":"..."}
             MetaRow r;
-            r.text      = extract_field(line, "text");
-            r.timestamp = extract_field(line, "ts");
+            if (j.contains("text")) {
+                r.text = j["text"].get<std::string>();
+            }
+            if (j.contains("ts")) {
+                r.timestamp = j["ts"].get<std::string>();
+            }
             rows_.push_back(std::move(r));
         }
     }
@@ -121,9 +78,10 @@ uint64_t MetadataStore::append(const char * text, const char * timestamp,
                                 std::string & err) {
     if (fd_ < 0) { err = "meta not open"; return UINT64_MAX; }
 
-    std::string t  = text      ? json_escape(text)      : "";
-    std::string ts = timestamp ? json_escape(timestamp)  : "";
-    std::string line = "{\"text\":\"" + t + "\",\"ts\":\"" + ts + "\"}\n";
+    json j;
+    j["text"] = text ? text : "";
+    j["ts"] = timestamp ? timestamp : "";
+    std::string line = j.dump() + "\n";
 
     ssize_t written = ::write(fd_, line.data(), line.size());
     if (written != (ssize_t)line.size()) {
@@ -147,7 +105,11 @@ bool MetadataStore::mark_deleted(uint64_t id, std::string & err) {
         return false;
     }
 
-    std::string line = "{\"op\":\"del\",\"id\":" + std::to_string(id) + "}\n";
+    json j;
+    j["op"] = "del";
+    j["id"] = id;
+    std::string line = j.dump() + "\n";
+
     ssize_t written = ::write(fd_, line.data(), line.size());
     if (written != (ssize_t)line.size()) {
         err = std::string("write tombstone: ") + strerror(errno);