ladybug 0.20.2.dev20260907__tar.gz → 0.20.2.dev20260908__tar.gz

sdist/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ladybug
-Version: 0.20.2.dev20260907
+Version: 0.20.2.dev20260908
 Summary: Highly scalable, extremely fast, easy-to-use embeddable graph database
 Home-page: https://github.com/LadybugDB/ladybug
 License: MIT

sdist/ladybug-source/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.15)
 
-project(Lbug VERSION 0.20.2.20260907 LANGUAGES CXX C)
+project(Lbug VERSION 0.20.2.20260908 LANGUAGES CXX C)
 
 option(SINGLE_THREADED "Single-threaded mode" FALSE)
 if(SINGLE_THREADED)

sdist/ladybug-source/src/include/processor/processor.h

@@ -1,15 +1,48 @@
 #pragma once
 
+#include <mutex>
+#include <thread>
+#include <unordered_map>
+
 #include "common/task_system/task_scheduler.h"
 
 namespace lbug {
 namespace main {
 class QueryResult;
 }
+namespace storage {
+class MemoryManager;
+}
 namespace processor {
 class FactorizedTable;
 class PhysicalPlan;
 class PhysicalOperator;
+class ResultSet;
+struct ResultSetDescriptor;
+
+// Per-database pool of reusable ResultSets, one slot per executing thread. A cached ResultSet
+// owns ValueVectors whose buffers belong to this database's MemoryManager, so the pool must be
+// owned by (and destroyed with) the database rather than living in a process-wide thread_local:
+// a thread_local slot outlives Database::~Database and later releases (or worse, reuses) buffers
+// through a dangling MemoryManager pointer when the next database runs a query on that thread.
+class ResultSetPool {
+public:
+    // Returns the ResultSet cached for the calling thread when its descriptor id matches,
+    // otherwise allocates a fresh one and caches it. The caller must keep the returned
+    // shared_ptr for as long as it uses the ResultSet: a nested query on the same thread may
+    // replace the slot, and the pool's own reference is dropped when this database closes.
+    std::shared_ptr<ResultSet> getOrCreate(ResultSetDescriptor* descriptor,
+        storage::MemoryManager* memoryManager);
+
+private:
+    struct Entry {
+        uint64_t descriptorID = UINT64_MAX;
+        std::shared_ptr<ResultSet> resultSet;
+    };
+    std::mutex mtx;
+    std::unordered_map<std::thread::id, Entry> entries;
+};
+
 class QueryProcessor {
 
 public:
@@ -20,6 +53,7 @@ public:
 #endif
 
     common::TaskScheduler* getTaskScheduler() { return taskScheduler.get(); }
+    ResultSetPool& getResultSetPool() { return resultSetPool; }
 
     std::unique_ptr<main::QueryResult> execute(PhysicalPlan* physicalPlan,
         ExecutionContext* context);
@@ -30,6 +64,10 @@ private:
     void initTask(common::Task* task);
 
 private:
+    // Declared before the scheduler so worker threads are joined before the pool is destroyed.
+    // The pool never outlives the buffers it caches because Database destroys its QueryProcessor
+    // before its MemoryManager.
+    ResultSetPool resultSetPool;
     std::unique_ptr<common::TaskScheduler> taskScheduler;
 };
 

sdist/ladybug-source/src/include/processor/result/result_set_descriptor.h

@@ -27,13 +27,13 @@ struct DataChunkDescriptor {
 
 struct LBUG_API ResultSetDescriptor {
     // Monotonically increasing identity that survives pointer reuse (ABA
-    // prevention).  Thread-local ResultSet caching in ProcessorTask::run()
+    // prevention).  Per-database ResultSet pooling in ProcessorTask::run()
     // compares this ID instead of the raw pointer so that a descriptor
     // allocated at the same address as a freed one is never confused with it.
     // copy() preserves the id: a copy semantically represents the same descriptor
     // slot (same plan position, same content). In particular the per-execution
     // re-attach of cached-plan sink descriptors (ClientContext::attachSinkDescriptors)
-    // must carry the same id across executions, otherwise the thread-local cache
+    // must carry the same id across executions, otherwise the per-database ResultSetPool
     // would miss on every execution of the same prepared statement.
     static inline std::atomic<uint64_t> nextID{0};
     uint64_t id;

sdist/ladybug-source/src/include/storage/table/arrow_node_table.h

@@ -13,6 +13,7 @@
 #include "storage/table/columnar_node_table_base.h"
 
 namespace lbug {
+struct ArrowTableData;
 namespace storage {
 
 struct ArrowNodeTableScanState final : ColumnarNodeTableScanState {
@@ -75,7 +76,7 @@ class ArrowNodeTable final : public ColumnarNodeTableBase {
 public:
     ArrowNodeTable(const StorageManager* storageManager,
         const catalog::NodeTableCatalogEntry* nodeTableEntry, MemoryManager* memoryManager,
-        ArrowSchemaWrapper schema, std::vector<ArrowArrayWrapper> arrays, std::string arrowId);
+        std::shared_ptr<ArrowTableData> arrowData, std::string arrowId);
 
     ~ArrowNodeTable();
 
@@ -121,6 +122,10 @@ private:
         const std::vector<int64_t>& outputToArrowColumnIdx) const;
 
 private:
+    // Pin on the registry entry: keeps the Arrow buffers alive even after
+    // unregisterArrowData() erases the registry map entry (issue #933).
+    // `schema`/`arrays` below are shallow non-owning views into `arrowData`.
+    std::shared_ptr<ArrowTableData> arrowData;
     ArrowSchemaWrapper schema;
     std::vector<ArrowArrayWrapper> arrays;
     std::vector<std::optional<common::ArrowLogicalTypeInfo>> columnLogicalTypeInfos;

sdist/ladybug-source/src/include/storage/table/arrow_rel_table.h

@@ -32,10 +32,7 @@ public:
     ArrowRelTable(catalog::RelGroupCatalogEntry* relGroupEntry, common::table_id_t fromTableID,
         common::table_id_t toTableID, const StorageManager* storageManager,
         MemoryManager* memoryManager, const NodeTable* fromNodeTable, const NodeTable* toNodeTable,
-        ArrowRelTableLayout layout, ArrowSchemaWrapper schema,
-        std::vector<ArrowArrayWrapper> arrays, ArrowSchemaWrapper indptrSchema,
-        std::vector<ArrowArrayWrapper> indptrArrays, std::string arrowId,
-        std::string dstColumnName = "to");
+        std::shared_ptr<ArrowRelTableData> arrowData, std::string arrowId);
     ~ArrowRelTable();
 
     void initScanState(transaction::Transaction* transaction, TableScanState& scanState,
@@ -78,6 +75,9 @@ private:
 
     const NodeTable* fromNodeTable;
     const NodeTable* toNodeTable;
+    // Pin on the registry entry (issue #933); `schema`/`arrays`/`indptrSchema`/
+    // `indptrArrays` below are shallow non-owning views into `arrowData`.
+    std::shared_ptr<ArrowRelTableData> arrowData;
     ArrowRelTableLayout layout;
     ArrowSchemaWrapper schema;
     std::vector<ArrowArrayWrapper> arrays;

sdist/ladybug-source/src/include/storage/table/arrow_table_support.h

@@ -22,6 +22,16 @@ struct ArrowRelTableData {
     std::string dstColumnName = "to";
 };
 
+// Registry-owned Arrow node-table payload. Stored as a shared_ptr in the
+// process-wide registry so that tables can pin the data they view: the
+// shared_ptr copy returned by getArrowData()/getArrowRelData() keeps the
+// underlying Arrow buffers alive even if another thread unregisters (erases)
+// the registry entry concurrently (CWE-416/667, issue #933).
+struct ArrowTableData {
+    ArrowSchemaWrapper schema;
+    std::vector<ArrowArrayWrapper> arrays;
+};
+
 // Result of creating an arrow table view
 struct ArrowTableCreationResult {
     std::unique_ptr<main::QueryResult> queryResult;
@@ -37,12 +47,15 @@ public:
     // Register Arrow relationship data and return an ID
     static std::string registerArrowRelData(ArrowRelTableData data);
 
-    // Retrieve Arrow data by ID (returns pointers to data in registry)
-    static bool getArrowData(const std::string& id, ArrowSchemaWrapper*& schema,
-        std::vector<ArrowArrayWrapper>*& arrays);
+    // Retrieve Arrow data by ID. Returns a shared_ptr pinning the registry
+    // entry's lifetime: the caller may safely dereference it after the
+    // registry lock is released, even if another thread concurrently
+    // unregisters the same ID. Returns nullptr when the ID is unknown.
+    static std::shared_ptr<ArrowTableData> getArrowData(const std::string& id);
 
-    // Retrieve Arrow relationship data by ID (returns pointer to data in registry)
-    static bool getArrowRelData(const std::string& id, ArrowRelTableData*& data);
+    // Retrieve Arrow relationship data by ID (same lifetime semantics as
+    // getArrowData above). Returns nullptr when the ID is unknown.
+    static std::shared_ptr<ArrowRelTableData> getArrowRelData(const std::string& id);
 
     // Unregister Arrow data by ID
     static void unregisterArrowData(const std::string& id);

sdist/ladybug-source/src/optimizer/optimizer.cpp

@@ -4,6 +4,7 @@
 #include <cstdlib>
 #include <unordered_set>
 
+#include "binder/expression/expression_util.h"
 #include "common/enums/extend_direction_util.h"
 #include "main/client_context.h"
 #include "optimizer/acc_hash_join_optimizer.h"
@@ -32,6 +33,14 @@
 #include "planner/operator/logical_filter.h"
 #include "planner/operator/logical_hash_join.h"
 #include "planner/operator/logical_operator.h"
+#include "planner/operator/logical_partitioner.h"
+#include "planner/operator/logical_unwind_deduplicate.h"
+#include "planner/operator/scan/logical_count_anti_edge_chain.h"
+#include "planner/operator/scan/logical_count_extend_chain.h"
+#include "planner/operator/scan/logical_count_rel_table.h"
+#include "planner/operator/scan/logical_query_primary_key_lookup.h"
+#include "planner/operator/scan/logical_reachable_count.h"
+#include "planner/operator/scan/logical_rel_degree_table.h"
 #include "planner/operator/scan/logical_scan_node_table.h"
 #include "transaction/transaction.h"
 
@@ -88,6 +97,64 @@ void dumpLogicalTree(const planner::LogicalOperator* op, int depth,
     } else if (op->getOperatorType() == planner::LogicalOperatorType::SCAN_NODE_TABLE) {
         fprintf(stderr, " [%s]",
             op->constCast<planner::LogicalScanNodeTable>().getNodeID()->toString().c_str());
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::COUNT_REL_TABLE) {
+        auto& count = op->constCast<planner::LogicalCountRelTable>();
+        fprintf(stderr, " [%s dir=%s bound=%s]", count.getRelGroupEntry()->getName().c_str(),
+            common::ExtendDirectionUtil::toString(count.getDirection()).c_str(),
+            count.getBoundNode()->getUniqueName().c_str());
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::COUNT_EXTEND_CHAIN) {
+        auto& count = op->constCast<planner::LogicalCountExtendChain>();
+        fprintf(stderr, " [hops=%llu tables=", (unsigned long long)count.getHops().size());
+        bool first = true;
+        for (auto& hop : count.getHops()) {
+            for (auto& spec : hop.relScans) {
+                fprintf(stderr, "%s%s", first ? "" : ",", spec.relTableName.c_str());
+                first = false;
+            }
+        }
+        fprintf(stderr, "]");
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::COUNT_ANTI_EDGE_CHAIN) {
+        auto& count = op->constCast<planner::LogicalCountAntiEdgeChain>();
+        fprintf(stderr,
+            " [%s antiDir=%s hops=%llu id<>=", count.getAntiRelEntry()->getName().c_str(),
+            common::ExtendDirectionUtil::toString(count.getAntiEdgeDir()).c_str(),
+            (unsigned long long)count.getSuffixHops().size());
+        fprintf(stderr, "%s]", count.getHasNotEquals() ? "true" : "false");
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::REACHABLE_COUNT) {
+        auto& count = op->constCast<planner::LogicalReachableCount>();
+        fprintf(stderr, " [%s dir=%s bound=%s nbr=%s range=%u..%u]",
+            count.getRelGroupEntry()->getName().c_str(),
+            common::ExtendDirectionUtil::toString(count.getDirection()).c_str(),
+            count.getBoundNode()->getUniqueName().c_str(),
+            count.getNbrNode()->getUniqueName().c_str(), count.getLowerBound(),
+            count.getUpperBound());
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::REL_DEGREE_TABLE) {
+        auto& degree = op->constCast<planner::LogicalRelDegreeTable>();
+        fprintf(stderr, " [%s dir=%s mode=%s]", degree.getRelGroupEntry()->getName().c_str(),
+            common::ExtendDirectionUtil::toString(degree.getDirection()).c_str(),
+            degree.getMode() == planner::RelDegreeTableMode::ACTIVE_BOUND_COUNT ?
+                "ACTIVE_BOUND_COUNT" :
+            degree.getMode() == planner::RelDegreeTableMode::TOP_K_DEGREES ? "TOP_K_DEGREES" :
+                                                                             "OFFSET_COUNT");
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::QUERY_PRIMARY_KEY_LOOKUP) {
+        auto& lookup = op->constCast<planner::LogicalQueryPrimaryKeyLookup>();
+        fprintf(stderr, " [table=%llu key=%s]", (unsigned long long)lookup.getTableID(),
+            lookup.getKey()->toString().c_str());
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::UNWIND_DEDUPLICATE) {
+        auto& dedup = op->constCast<planner::LogicalUnwindDeduplicate>();
+        fprintf(stderr, " [keys=%s]",
+            binder::ExpressionUtil::toString(dedup.getKeyExpressions()).c_str());
+    } else if (op->getOperatorType() == planner::LogicalOperatorType::PARTITIONER) {
+        fprintf(stderr, " [keys=%llu]",
+            (unsigned long long)op->constCast<planner::LogicalPartitioner>()
+                .getInfo()
+                .getNumInfos());
+    } else {
+        // Generic fallback for operators without a dedicated handler above.
+        auto exprs = op->getExpressionsForPrinting();
+        if (!exprs.empty()) {
+            fprintf(stderr, " [%s]", exprs.c_str());
+        }
     }
     fprintf(stderr, "\n");
     for (auto i = 0u; i < op->getNumChildren(); ++i) {

sdist/ladybug-source/src/planner/operator/logical_operator.cpp

@@ -24,6 +24,8 @@ std::string LogicalOperatorUtils::logicalOperatorTypeToString(LogicalOperatorTyp
         return "COPY_FROM";
     case LogicalOperatorType::COPY_TO:
         return "COPY_TO";
+    case LogicalOperatorType::COUNT_ANTI_EDGE_CHAIN:
+        return "COUNT_ANTI_EDGE_CHAIN";
     case LogicalOperatorType::COUNT_EXTEND_CHAIN:
         return "COUNT_EXTEND_CHAIN";
     case LogicalOperatorType::COUNT_REL_TABLE:
@@ -132,9 +134,11 @@ std::string LogicalOperatorUtils::logicalOperatorTypeToString(LogicalOperatorTyp
         return "EXTENSION_CLAUSE";
     case LogicalOperatorType::UNWIND_DEDUPLICATE:
         return "UNWIND_DEDUPLICATE";
-    default:
-        throw RuntimeException("Unknown logical operator type.");
     }
+    // No default label: -Wswitch then enforces that every LogicalOperatorType enumerator is
+    // handled above, so adding a new operator fails the build here instead of failing at
+    // runtime. The throw only guards against out-of-range values (e.g. deserialized data).
+    throw RuntimeException("Unknown logical operator type.");
 }
 // LCOV_EXCL_STOP
 

sdist/ladybug-source/src/processor/processor.cpp

@@ -6,6 +6,7 @@
 #include "processor/operator/sink.h"
 #include "processor/physical_plan.h"
 #include "processor/processor_task.h"
+#include "processor/result/result_set.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -13,6 +14,37 @@ using namespace lbug::storage;
 namespace lbug {
 namespace processor {
 
+std::shared_ptr<ResultSet> ResultSetPool::getOrCreate(ResultSetDescriptor* descriptor,
+    MemoryManager* memoryManager) {
+    const auto threadID = std::this_thread::get_id();
+    std::shared_ptr<ResultSet> reusable;
+    {
+        std::lock_guard lock{mtx};
+        auto& entry = entries[threadID];
+        if (entry.resultSet && entry.descriptorID == descriptor->id) {
+            reusable = entry.resultSet;
+        }
+    }
+    if (reusable) {
+        // Same prepared statement on this thread: reuse the allocation. Only this thread touches
+        // its slot's ResultSet, so the reset happens after the lock is released.
+        reusable->resetForReuse();
+        return reusable;
+    }
+    // First time on this thread, or a different prepared statement: allocate fresh outside the
+    // lock, then swap it in and release the previous ResultSet after unlocking.
+    auto resultSet = std::make_shared<ResultSet>(descriptor, memoryManager);
+    std::shared_ptr<ResultSet> previous;
+    {
+        std::lock_guard lock{mtx};
+        auto& entry = entries[threadID];
+        previous = std::move(entry.resultSet);
+        entry.resultSet = resultSet;
+        entry.descriptorID = descriptor->id;
+    }
+    return resultSet;
+}
+
 #if defined(__APPLE__)
 QueryProcessor::QueryProcessor(uint64_t numThreads, uint32_t threadQos) {
     taskScheduler = std::make_unique<TaskScheduler>(numThreads, threadQos);

sdist/ladybug-source/src/processor/processor_task.cpp

@@ -2,8 +2,10 @@
 
 #include "common/task_system/progress_bar.h"
 #include "main/client_context.h"
+#include "main/database.h"
 #include "main/settings.h"
 #include "processor/execution_context.h"
+#include "processor/processor.h"
 #include "processor/result/result_set.h"
 #include "storage/buffer_manager/memory_manager.h"
 
@@ -33,30 +35,21 @@ void ProcessorTask::run() {
     //     for i in range(n): conn.execute("RETURN $i", {"i": i})
     // that's a 16KB+ calloc on every iteration.
     //
-    // We instead keep a thread-local shared_ptr<ResultSet> whose descriptor
-    // matches this sink's, so each thread of a multi-threaded task gets
-    // its own allocation-free slot. The thread owns the ResultSet outright
-    // (no aliasing), so the lifetime is straightforward: dropped when the
-    // thread exits, or when a different prepared statement runs on this
-    // thread and the descriptor pointer no longer matches.
+    // The reusable ResultSet is cached per executing thread in the database-owned
+    // ResultSetPool (see QueryProcessor), never in a process-wide thread_local: the cached
+    // vectors' buffers belong to this database's MemoryManager, and a thread_local slot would
+    // outlive Database::~Database and free (or reuse) them through a dangling MemoryManager the
+    // next time any database runs a query on this thread. Holding the shared_ptr here keeps the
+    // ResultSet alive even if a nested query on this thread replaces the slot.
     ResultSet* resultSetPtr = nullptr;
     std::unique_ptr<ResultSet> ownedResultSet;
+    std::shared_ptr<ResultSet> pooledResultSet;
+    auto* clientContext = executionContext->clientContext;
     if (auto* desc = sink->getDescriptor()) {
-        thread_local uint64_t cachedDescID = UINT64_MAX;
-        thread_local std::shared_ptr<processor::ResultSet> cachedResultSet;
-        if (cachedDescID == desc->id && cachedResultSet) {
-            // Same prepared statement on this thread: reuse the allocation.
-            cachedResultSet->resetForReuse();
-            resultSetPtr = cachedResultSet.get();
-        } else {
-            // First time on this thread, or a different prepared statement:
-            // allocate fresh. Owning shared_ptr; lifetime ends with the
-            // thread (or when the descriptor changes).
-            cachedResultSet = std::make_shared<processor::ResultSet>(desc,
-                storage::MemoryManager::Get(*executionContext->clientContext));
-            cachedDescID = desc->id;
-            resultSetPtr = cachedResultSet.get();
-        }
+        pooledResultSet =
+            clientContext->getDatabase()->getQueryProcessor()->getResultSetPool().getOrCreate(desc,
+                storage::MemoryManager::Get(*clientContext));
+        resultSetPtr = pooledResultSet.get();
     } else {
         // No descriptor (e.g. OrderByMerge): fall back to per-call allocation.
         ownedResultSet =

sdist/ladybug-source/src/processor/result/result_set_descriptor.cpp

@@ -28,8 +28,8 @@ std::unique_ptr<ResultSetDescriptor> ResultSetDescriptor::copy() const {
         std::make_unique<ResultSetDescriptor>(std::move(dataChunkDescriptorsCopy));
     // A copy is the same descriptor slot with identical content (copies are never mutated
     // afterwards), so it inherits the source's identity. This keeps the id stable across
-    // executions of a prepared statement's cached plan, letting the thread-local ResultSet
-    // cache in ProcessorTask::run() actually hit. Two live descriptors with the same id are
+    // executions of a prepared statement's cached plan, letting the per-database ResultSetPool
+    // used by ProcessorTask::run() actually hit. Two live descriptors with the same id are
     // by construction content-identical, so cache reuse remains safe.
     descriptorCopy->id = id;
     return descriptorCopy;

sdist/ladybug-source/src/storage/local_storage/local_node_table.cpp

@@ -3,6 +3,7 @@
 #include "catalog/catalog_entry/node_table_catalog_entry.h"
 #include "common/cast.h"
 #include "common/exception/message.h"
+#include "common/exception/runtime.h"
 #include "common/types/types.h"
 #include "common/types/value/value.h"
 #include "storage/index/hash_index.h"
@@ -42,9 +43,19 @@ void LocalNodeTable::initLocalHashIndex(MemoryManager& mm) {
 }
 
 bool LocalNodeTable::isVisible(const Transaction* transaction, offset_t offset) const {
+    if (offset < startOffset) {
+        throw RuntimeException(
+            "Primary-key index contains an invalid local node offset. Please drop and rebuild "
+            "the _PK index.");
+    }
     auto [nodeGroupIdx, offsetInGroup] =
         StorageUtils::getNodeGroupIdxAndOffsetInChunk(offset - startOffset);
-    auto* nodeGroup = nodeGroups.getNodeGroup(nodeGroupIdx);
+    auto* nodeGroup = nodeGroups.getNodeGroup(nodeGroupIdx, true /*mayOutOfBound*/);
+    if (nodeGroup == nullptr || offsetInGroup >= nodeGroup->getNumRows()) {
+        throw RuntimeException(
+            "Primary-key index contains an invalid local node offset. Please drop and rebuild "
+            "the _PK index.");
+    }
     if (nodeGroup->isDeleted(transaction, offsetInGroup)) {
         return false;
     }

sdist/ladybug-source/src/storage/storage_manager.cpp

@@ -211,24 +211,19 @@ void StorageManager::createNodeTable(NodeTableCatalogEntry* entry, main::ClientC
             // Extract Arrow ID from storage string
             std::string arrowId = entry->getStorage().substr(8);
 
-            // Retrieve Arrow data from registry (as pointers to registry data)
-            ArrowSchemaWrapper* schema = nullptr;
-            std::vector<ArrowArrayWrapper>* arrays = nullptr;
-            if (!ArrowTableSupport::getArrowData(arrowId, schema, arrays)) {
+            // Retrieve Arrow data from registry. The returned shared_ptr pins
+            // the entry's lifetime past the registry lock, so a concurrent
+            // DROP TABLE / unregister cannot free the buffers while we build
+            // the table's shallow views below (issue #933).
+            auto arrowData = ArrowTableSupport::getArrowData(arrowId);
+            if (!arrowData) {
                 throw common::RuntimeException("Failed to retrieve Arrow data for ID: " + arrowId);
             }
 
-            // Create wrappers that reference registry memory while registry keeps ownership.
-            ArrowSchemaWrapper schemaCopy = createShallowCopy(*schema);
-            std::vector<ArrowArrayWrapper> arraysCopy;
-            arraysCopy.reserve(arrays->size());
-            for (const auto& arr : *arrays) {
-                arraysCopy.push_back(createShallowCopy(arr));
-            }
-
-            // Create Arrow-backed node table
+            // Create Arrow-backed node table (it keeps its own pin plus
+            // shallow non-owning views into the pinned data).
             tables[entry->getTableID()] = std::make_unique<ArrowNodeTable>(this, entry,
-                &memoryManager, std::move(schemaCopy), std::move(arraysCopy), arrowId);
+                &memoryManager, std::move(arrowData), arrowId);
         } else {
             throw common::RuntimeException(
                 "Unsupported storage option for node table: " + entry->getStorage());
@@ -262,8 +257,8 @@ void StorageManager::addRelTable(RelGroupCatalogEntry* entry, const RelTableCata
     } else if (!entry->getStorage().empty()) {
         if (entry->getStorage().substr(0, 8) == "arrow://") {
             std::string arrowId = entry->getStorage().substr(8);
-            ArrowRelTableData* relData = nullptr;
-            if (!ArrowTableSupport::getArrowRelData(arrowId, relData)) {
+            auto relData = ArrowTableSupport::getArrowRelData(arrowId);
+            if (!relData) {
                 throw common::RuntimeException("Failed to retrieve Arrow data for ID: " + arrowId);
             }
             if (!tables.contains(info.nodePair.srcTableID) ||
@@ -277,23 +272,9 @@ void StorageManager::addRelTable(RelGroupCatalogEntry* entry, const RelTableCata
                 throw common::RuntimeException(
                     "Arrow rel table currently supports only regular node tables");
             }
-            ArrowSchemaWrapper schemaCopy = createShallowCopy(relData->schema);
-            std::vector<ArrowArrayWrapper> arraysCopy;
-            arraysCopy.reserve(relData->arrays.size());
-            for (const auto& arr : relData->arrays) {
-                arraysCopy.push_back(createShallowCopy(arr));
-            }
-            ArrowSchemaWrapper indptrSchemaCopy = createShallowCopy(relData->indptrSchema);
-            std::vector<ArrowArrayWrapper> indptrArraysCopy;
-            indptrArraysCopy.reserve(relData->indptrArrays.size());
-            for (const auto& arr : relData->indptrArrays) {
-                indptrArraysCopy.push_back(createShallowCopy(arr));
-            }
             tables[info.oid] = std::make_unique<ArrowRelTable>(entry, info.nodePair.srcTableID,
                 info.nodePair.dstTableID, this, &memoryManager, fromNodeTable, toNodeTable,
-                relData->layout, std::move(schemaCopy), std::move(arraysCopy),
-                std::move(indptrSchemaCopy), std::move(indptrArraysCopy), arrowId,
-                relData->dstColumnName);
+                std::move(relData), arrowId);
         } else {
             throw common::RuntimeException(
                 "Unsupported storage option for rel table: " + entry->getStorage());

sdist/ladybug-source/src/storage/table/arrow_node_table.cpp

@@ -41,11 +41,21 @@ static std::vector<std::optional<common::ArrowLogicalTypeInfo>> resolveColumnLog
 
 ArrowNodeTable::ArrowNodeTable(const StorageManager* storageManager,
     const catalog::NodeTableCatalogEntry* nodeTableEntry, MemoryManager* memoryManager,
-    ArrowSchemaWrapper schema, std::vector<ArrowArrayWrapper> arrays, std::string arrowId)
+    std::shared_ptr<ArrowTableData> arrowData, std::string arrowId)
     : ColumnarNodeTableBase{storageManager, nodeTableEntry, memoryManager,
           std::make_unique<ArrowNodeTableScanSharedState>(scanMorselSize)},
-      schema{std::move(schema)}, arrays{std::move(arrays)}, totalRows{0},
-      arrowId{std::move(arrowId)} {
+      arrowData{std::move(arrowData)}, totalRows{0}, arrowId{std::move(arrowId)} {
+    if (!this->arrowData) {
+        throw common::RuntimeException("Arrow data is null");
+    }
+    // Shallow non-owning views into the pinned registry data. Safe because
+    // `arrowData` keeps the underlying Arrow buffers alive for the table's
+    // lifetime, even after the registry entry is erased.
+    this->schema = createShallowCopy(this->arrowData->schema);
+    this->arrays.reserve(this->arrowData->arrays.size());
+    for (const auto& array : this->arrowData->arrays) {
+        this->arrays.push_back(createShallowCopy(array));
+    }
     // Note: release may be nullptr if schema is managed by registry
     if (!this->schema.format) {
         throw common::RuntimeException("Arrow schema format cannot be null");

sdist/ladybug-source/src/storage/table/arrow_rel_table.cpp

@@ -108,14 +108,27 @@ void ArrowRelTableScanState::setToTable(const transaction::Transaction* transact
 
 ArrowRelTable::ArrowRelTable(catalog::RelGroupCatalogEntry* relGroupEntry, table_id_t fromTableID,
     table_id_t toTableID, const StorageManager* storageManager, MemoryManager* memoryManager,
-    const NodeTable* fromNodeTable, const NodeTable* toNodeTable, ArrowRelTableLayout layout,
-    ArrowSchemaWrapper schema, std::vector<ArrowArrayWrapper> arrays,
-    ArrowSchemaWrapper indptrSchema, std::vector<ArrowArrayWrapper> indptrArrays,
-    std::string arrowId, std::string dstColumnName)
+    const NodeTable* fromNodeTable, const NodeTable* toNodeTable,
+    std::shared_ptr<ArrowRelTableData> arrowData, std::string arrowId)
     : ColumnarRelTableBase{relGroupEntry, fromTableID, toTableID, storageManager, memoryManager},
-      fromNodeTable{fromNodeTable}, toNodeTable{toNodeTable}, layout{layout},
-      schema{std::move(schema)}, arrays{std::move(arrays)}, indptrSchema{std::move(indptrSchema)},
-      indptrArrays{std::move(indptrArrays)}, arrowId{std::move(arrowId)} {
+      fromNodeTable{fromNodeTable}, toNodeTable{toNodeTable}, arrowData{std::move(arrowData)},
+      arrowId{std::move(arrowId)} {
+    if (!this->arrowData) {
+        throw RuntimeException("Arrow relationship data is null");
+    }
+    // Shallow non-owning views into the pinned registry data (see ArrowNodeTable).
+    this->layout = this->arrowData->layout;
+    const std::string dstColumnName = this->arrowData->dstColumnName;
+    this->schema = createShallowCopy(this->arrowData->schema);
+    this->arrays.reserve(this->arrowData->arrays.size());
+    for (const auto& array : this->arrowData->arrays) {
+        this->arrays.push_back(createShallowCopy(array));
+    }
+    this->indptrSchema = createShallowCopy(this->arrowData->indptrSchema);
+    this->indptrArrays.reserve(this->arrowData->indptrArrays.size());
+    for (const auto& array : this->arrowData->indptrArrays) {
+        this->indptrArrays.push_back(createShallowCopy(array));
+    }
     if (!this->schema.format) {
         throw RuntimeException("Arrow schema format cannot be null");
     }

sdist/ladybug-source/src/storage/table/arrow_table_support.cpp

@@ -12,16 +12,18 @@ namespace lbug {
 
 // Global registry for Arrow table data
 // Memory Management:
-// - Registry owns the Arrow data (ArrowSchemaWrapper/ArrowArrayWrapper with release callbacks)
-// - Arrow-backed tables store shallow copies (no release callbacks) and the arrowId
+// - Registry owns the Arrow data via shared_ptr (ArrowSchemaWrapper/ArrowArrayWrapper with
+//   release callbacks). getArrowData()/getArrowRelData() return a shared_ptr copy taken under
+//   the mutex, so callers pin the data past the lock scope; erasing the map entry only drops
+//   the registry's reference while live tables keep buffers valid (issue #933).
+// - Arrow-backed tables hold a shared_ptr pin plus shallow copies (no release callbacks) and
+//   the arrowId
 // - When a table is dropped (via DROP TABLE or unregisterArrowTable), the table's
 //   destructor automatically calls unregisterArrowData to clean up the registry entry
 // - The wrappers' destructors call the release callbacks to free the actual Arrow memory
 static std::mutex g_arrowRegistryMutex;
-static std::unordered_map<std::string,
-    std::pair<ArrowSchemaWrapper, std::vector<ArrowArrayWrapper>>>
-    g_arrowRegistry;
-static std::unordered_map<std::string, ArrowRelTableData> g_arrowRelRegistry;
+static std::unordered_map<std::string, std::shared_ptr<ArrowTableData>> g_arrowRegistry;
+static std::unordered_map<std::string, std::shared_ptr<ArrowRelTableData>> g_arrowRelRegistry;
 
 std::string join(const std::vector<std::string>& strings, const std::string& delimiter) {
     if (strings.empty())
@@ -58,8 +60,11 @@ std::string ArrowTableSupport::registerArrowData(ArrowSchemaWrapper schema,
     static size_t nextId = 0;
     std::string id = "arrow_" + std::to_string(nextId++);
 
-    // Store in registry
-    g_arrowRegistry[id] = std::make_pair(std::move(schema), std::move(arrays));
+    // Store in registry (shared ownership so live tables can pin the data).
+    auto entry = std::make_shared<ArrowTableData>();
+    entry->schema = std::move(schema);
+    entry->arrays = std::move(arrays);
+    g_arrowRegistry[id] = std::move(entry);
 
     return id;
 }
@@ -69,34 +74,33 @@ std::string ArrowTableSupport::registerArrowRelData(ArrowRelTableData data) {
 
     static size_t nextRelId = 0;
     std::string id = "arrow_rel_" + std::to_string(nextRelId++);
-    g_arrowRelRegistry[id] = std::move(data);
+    auto entry = std::make_shared<ArrowRelTableData>(std::move(data));
+    g_arrowRelRegistry[id] = std::move(entry);
     return id;
 }
 
-bool ArrowTableSupport::getArrowData(const std::string& id, ArrowSchemaWrapper*& schema,
-    std::vector<ArrowArrayWrapper>*& arrays) {
+std::shared_ptr<ArrowTableData> ArrowTableSupport::getArrowData(const std::string& id) {
     std::lock_guard<std::mutex> lock(g_arrowRegistryMutex);
 
     auto it = g_arrowRegistry.find(id);
     if (it == g_arrowRegistry.end()) {
-        return false;
+        return nullptr;
     }
 
-    // Return pointers to the data in the registry (not copies)
-    schema = &it->second.first;
-    arrays = &it->second.second;
-    return true;
+    // Copy the shared_ptr under the lock: the caller pins the data's lifetime
+    // past the lock scope, so a concurrent unregisterArrowData() (erase)
+    // cannot free the buffers out from under it.
+    return it->second;
 }
 
-bool ArrowTableSupport::getArrowRelData(const std::string& id, ArrowRelTableData*& data) {
+std::shared_ptr<ArrowRelTableData> ArrowTableSupport::getArrowRelData(const std::string& id) {
     std::lock_guard<std::mutex> lock(g_arrowRegistryMutex);
 
     auto it = g_arrowRelRegistry.find(id);
     if (it == g_arrowRelRegistry.end()) {
-        return false;
+        return nullptr;
     }
-    data = &it->second;
-    return true;
+    return it->second;
 }
 
 void ArrowTableSupport::unregisterArrowData(const std::string& id) {

sdist/ladybug-source/src/storage/table/node_table.cpp

@@ -877,6 +877,9 @@ bool NodeTable::isVisibleNoLock(const Transaction* transaction, offset_t offset)
         return false;
     }
     const auto* nodeGroup = getNodeGroupNoLock(nodeGroupIdx);
+    if (nodeGroup == nullptr) {
+        return false;
+    }
     return nodeGroup->isVisibleNoLock(transaction, offsetInGroup);
 }
 
@@ -928,7 +931,32 @@ bool NodeTable::lookupPK(const Transaction* transaction, ValueVector* keyVector,
     }
     if (auto* pkIndex = tryGetPrimaryKeyIndex()) {
         return pkIndex->lookupPrimaryKey(transaction, keyVector, vectorPos, result,
-            [&](offset_t offset) { return isVisibleNoLock(transaction, offset); });
+            [&](offset_t offset) {
+                // A stale/corrupt PK index entry must not reach isVisibleNoLock(), where an
+                // invalid node offset could otherwise be dereferenced in a release build. Keep
+                // this validation O(1) on the normal lookup path and fail the transaction instead
+                // of treating a corrupt entry as a missing key.
+                if (offset == INVALID_OFFSET) {
+                    throw RuntimeException(
+                        "Primary-key index contains an invalid node offset. Please drop and "
+                        "rebuild the _PK index.");
+                }
+                const auto nodeGroupIdx = StorageUtils::getNodeGroupIdx(offset);
+                if (nodeGroupIdx >= nodeGroups->getNumNodeGroupsNoLock()) {
+                    throw RuntimeException(
+                        "Primary-key index points to a missing node group. Please drop and "
+                        "rebuild the _PK index.");
+                }
+                const auto* nodeGroup = getNodeGroupNoLock(nodeGroupIdx);
+                const auto offsetInGroup =
+                    offset - StorageUtils::getStartOffsetOfNodeGroup(nodeGroupIdx);
+                if (nodeGroup == nullptr || offsetInGroup >= nodeGroup->getNumRows()) {
+                    throw RuntimeException(
+                        "Primary-key index contains an out-of-range node offset. Please drop "
+                        "and rebuild the _PK index.");
+                }
+                return isVisibleNoLock(transaction, offset);
+            });
     }
     auto keyToLookup = keyVector->getAsValue(vectorPos);
     ColumnPredicateSet predicateSet;

sdist/ladybug.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ladybug
-Version: 0.20.2.dev20260907
+Version: 0.20.2.dev20260908
 Summary: Highly scalable, extremely fast, easy-to-use embeddable graph database
 Home-page: https://github.com/LadybugDB/ladybug
 License: MIT

sdist/pyproject.toml

@@ -4,7 +4,7 @@ description = "Highly scalable, extremely fast, easy-to-use embeddable graph dat
 readme = "README.md"
 license = { text = "MIT" }
 keywords = ["graph", "database"]
-version = "0.20.2.dev20260907"
+version = "0.20.2.dev20260908"
 requires-python = ">=3.10,<3.15"
 
 [project.urls]