ladybug 0.22.0.dev20261022__tar.gz → 0.22.0.dev20261024__tar.gz

sdist/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ladybug
-Version: 0.22.0.dev20261022
+Version: 0.22.0.dev20261024
 Summary: Highly scalable, extremely fast, easy-to-use embeddable graph database
 Home-page: https://github.com/LadybugDB/ladybug
 License: MIT

sdist/ladybug-source/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.15)
 
-project(Lbug VERSION 0.22.0.20261022 LANGUAGES CXX C)
+project(Lbug VERSION 0.22.0.20261024 LANGUAGES CXX C)
 
 option(SINGLE_THREADED "Single-threaded mode" FALSE)
 if(SINGLE_THREADED)

sdist/ladybug-source/src/binder/ddl/property_definition.cpp

@@ -15,6 +15,16 @@ PropertyDefinition::PropertyDefinition(ColumnDefinition columnDefinition)
     defaultExpr = std::make_unique<ParsedLiteralExpression>(Value::createNullValue(), "NULL");
 }
 
+bool PropertyDefinition::hasDefault() const {
+    if (defaultExpr == nullptr) {
+        return false;
+    }
+    if (defaultExpr->getExpressionType() == common::ExpressionType::LITERAL) {
+        return !defaultExpr->constCast<parser::ParsedLiteralExpression>().getValue().isNull();
+    }
+    return !defaultExpr->getRawName().empty();
+}
+
 void PropertyDefinition::serialize(Serializer& serializer) const {
     serializer.serializeValue(columnDefinition.name);
     columnDefinition.type.serialize(serializer);

sdist/ladybug-source/src/c_api/value.cpp

@@ -3,6 +3,7 @@
 #include "c_api/helpers.h"
 #include "c_api/lbug.h"
 #include "common/constants.h"
+#include "common/type_utils.h"
 #include "common/types/types.h"
 #include "common/types/value/nested.h"
 #include "common/types/value/node.h"
@@ -40,7 +41,7 @@ static bool canCopyValueForNested(const Value& value, const LogicalType& targetT
     if (value.getDataType() == targetType) {
         return true;
     }
-    if (value.isNull() && value.getDataType().getLogicalTypeID() == LogicalTypeID::ANY) {
+    if (value.isNull()) {
         return true;
     }
     if (isEmptyNestedWithAny(value)) {
@@ -51,8 +52,45 @@ static bool canCopyValueForNested(const Value& value, const LogicalType& targetT
         targetType.getLogicalTypeID() == LogicalTypeID::FLOAT) {
         return true;
     }
-    return value.getDataType().getLogicalTypeID() == targetType.getLogicalTypeID() &&
-           value.getDataType().containsAny();
+    if (LogicalTypeUtils::isIntegral(value.getDataType()) &&
+        LogicalTypeUtils::isIntegral(targetType)) {
+        return true;
+    }
+    if (value.getDataType().getLogicalTypeID() != targetType.getLogicalTypeID()) {
+        return false;
+    }
+    const auto* nested = &value;
+    if (targetType.getLogicalTypeID() == LogicalTypeID::LIST) {
+        const auto& childType = ListType::getChildType(targetType);
+        for (auto i = 0u; i < NestedVal::getChildrenSize(nested); ++i) {
+            if (!canCopyValueForNested(*NestedVal::getChildVal(nested, i), childType)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    if (targetType.getLogicalTypeID() == LogicalTypeID::MAP) {
+        const auto& keyType = MapType::getKeyType(targetType);
+        const auto& valueType = MapType::getValueType(targetType);
+        for (auto i = 0u; i < NestedVal::getChildrenSize(nested); ++i) {
+            auto* mapEntry = NestedVal::getChildVal(nested, i);
+            if (!canCopyValueForNested(*NestedVal::getChildVal(mapEntry, 0), keyType) ||
+                !canCopyValueForNested(*NestedVal::getChildVal(mapEntry, 1), valueType)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    if (targetType.getLogicalTypeID() == LogicalTypeID::STRUCT) {
+        for (auto i = 0u; i < NestedVal::getChildrenSize(nested); ++i) {
+            if (!canCopyValueForNested(*NestedVal::getChildVal(nested, i),
+                    StructType::getFieldType(targetType, static_cast<struct_field_idx_t>(i)))) {
+                return false;
+            }
+        }
+        return true;
+    }
+    return value.getDataType().containsAny();
 }
 
 static std::string incompatibleParameterTypeMessage(const LogicalType& left,
@@ -69,28 +107,54 @@ static lbug_value* createCAPIValue(std::unique_ptr<Value> value) {
     return c_value;
 }
 
+// The target comes from LogicalTypeUtils::tryGetMaxLogicalType, which joins two integer types only
+// into one that holds both: implicit integer casts only widen, and a signed/unsigned pair becomes a
+// wider signed type. So the cast below is exact. CreateListAllIntegerPairs pins this.
+static std::unique_ptr<Value> castIntegerForNested(const Value& value,
+    const LogicalType& targetType) {
+    // A null of the target type keeps its exact type: SERIAL's default value is an INT64.
+    auto result = std::make_unique<Value>(Value::createNullValue(targetType));
+    result->setNull(false);
+    TypeUtils::visit(
+        value.getDataType(),
+        [&]<IntegerTypes SRC>(SRC) {
+            TypeUtils::visit(
+                targetType,
+                [&]<IntegerTypes DST>(DST) {
+                    auto castValue = static_cast<DST>(value.getValue<SRC>());
+                    result->copyFromRowLayout(reinterpret_cast<const uint8_t*>(&castValue));
+                },
+                [](auto) { UNREACHABLE_CODE; });
+        },
+        [](auto) { UNREACHABLE_CODE; });
+    return result;
+}
+
 static std::unique_ptr<Value> copyValueForNested(const Value& value,
     const LogicalType& targetType) {
-    if (value.isNull() && value.getDataType().getLogicalTypeID() == LogicalTypeID::ANY) {
+    if (value.getDataType() == targetType) {
+        return value.copy();
+    }
+    if (value.isNull()) {
         return std::make_unique<Value>(Value::createNullValue(targetType));
     }
-    if (value.getDataType() != targetType && isEmptyNestedWithAny(value)) {
+    if (isEmptyNestedWithAny(value)) {
         return std::make_unique<Value>(Value::createDefaultValue(targetType));
     }
-    if (value.getDataType() != targetType &&
-        targetType.getLogicalTypeID() == LogicalTypeID::STRING) {
+    if (targetType.getLogicalTypeID() == LogicalTypeID::STRING) {
         return std::make_unique<Value>(Value::createValue<std::string>(value.toString()));
     }
-    if (value.getDataType() != targetType &&
-        targetType.getLogicalTypeID() == LogicalTypeID::DOUBLE) {
+    if (targetType.getLogicalTypeID() == LogicalTypeID::DOUBLE) {
         return std::make_unique<Value>(std::stod(value.toString()));
     }
-    if (value.getDataType() != targetType &&
-        targetType.getLogicalTypeID() == LogicalTypeID::FLOAT) {
+    if (targetType.getLogicalTypeID() == LogicalTypeID::FLOAT) {
         return std::make_unique<Value>(static_cast<float>(std::stof(value.toString())));
     }
-    if (value.getDataType() != targetType &&
-        value.getDataType().getLogicalTypeID() == LogicalTypeID::LIST &&
+    if (LogicalTypeUtils::isIntegral(value.getDataType()) &&
+        LogicalTypeUtils::isIntegral(targetType)) {
+        return castIntegerForNested(value, targetType);
+    }
+    if (value.getDataType().getLogicalTypeID() == LogicalTypeID::LIST &&
         targetType.getLogicalTypeID() == LogicalTypeID::LIST) {
         std::vector<std::unique_ptr<Value>> children;
         const auto& childType = ListType::getChildType(targetType);
@@ -100,8 +164,7 @@ static std::unique_ptr<Value> copyValueForNested(const Value& value,
         }
         return std::make_unique<Value>(targetType.copy(), std::move(children));
     }
-    if (value.getDataType() != targetType &&
-        value.getDataType().getLogicalTypeID() == LogicalTypeID::MAP &&
+    if (value.getDataType().getLogicalTypeID() == LogicalTypeID::MAP &&
         targetType.getLogicalTypeID() == LogicalTypeID::MAP) {
         std::vector<std::unique_ptr<Value>> children;
         const auto& keyType = MapType::getKeyType(targetType);
@@ -121,6 +184,15 @@ static std::unique_ptr<Value> copyValueForNested(const Value& value,
         }
         return std::make_unique<Value>(targetType.copy(), std::move(children));
     }
+    if (value.getDataType().getLogicalTypeID() == LogicalTypeID::STRUCT &&
+        targetType.getLogicalTypeID() == LogicalTypeID::STRUCT) {
+        std::vector<std::unique_ptr<Value>> children;
+        for (auto i = 0u; i < NestedVal::getChildrenSize(&value); ++i) {
+            children.push_back(copyValueForNested(*NestedVal::getChildVal(&value, i),
+                StructType::getFieldType(targetType, static_cast<struct_field_idx_t>(i))));
+        }
+        return std::make_unique<Value>(targetType.copy(), std::move(children));
+    }
     return value.copy();
 }
 

sdist/ladybug-source/src/catalog/catalog.cpp

@@ -15,6 +15,7 @@
 #include "common/partition_routing_hook.h"
 #include "common/serializer/deserializer.h"
 #include "common/serializer/serializer.h"
+#include "common/string_utils.h"
 #include "extension/extension_manager.h"
 #include "function/function_collection.h"
 #include "main/client_context.h"
@@ -587,7 +588,7 @@ std::vector<FunctionCatalogEntry*> Catalog::getFunctionEntries(
 }
 
 bool Catalog::containsMacro(const Transaction* transaction, const std::string& macroName) const {
-    return macros->containsEntry(transaction, macroName);
+    return macros->containsEntry(transaction, StringUtils::getUpper(macroName));
 }
 
 function::ScalarMacroFunction* Catalog::getScalarMacroFunction(const Transaction* transaction,
@@ -627,11 +628,12 @@ std::vector<std::string> Catalog::getMacroNames(const Transaction* transaction)
 }
 
 void Catalog::dropMacro(Transaction* transaction, std::string& name) {
-    if (!containsMacro(transaction, name)) {
+    auto macroName = StringUtils::getUpper(name);
+    if (!macros->containsEntry(transaction, macroName)) {
         throw CatalogException{std::format("Macro {} doesn't exist.", name)};
     }
-    auto entry = getFunctionEntry(transaction, name);
-    macros->dropEntry(transaction, name, entry->getOID());
+    auto entry = macros->getEntry(transaction, macroName);
+    macros->dropEntry(transaction, macroName, entry->getOID());
 }
 
 void Catalog::registerBuiltInFunctions() {

sdist/ladybug-source/src/catalog/property_definition_collection.cpp

@@ -134,7 +134,11 @@ std::string PropertyDefinitionCollection::toCypher() const {
         if (typeStr.find("MAP") != std::string::npos) {
             StringUtils::replaceAll(typeStr, "  ", ",");
         }
-        ss << common::StringUtils::quoteIdentifier(def.getName()) << " " << typeStr << ",";
+        ss << common::StringUtils::quoteIdentifier(def.getName()) << " " << typeStr;
+        if (dataType.getLogicalTypeID() != LogicalTypeID::SERIAL && def.hasDefault()) {
+            ss << " DEFAULT " << def.getDefaultExpressionName();
+        }
+        ss << ",";
     }
     return ss.str();
 }

sdist/ladybug-source/src/common/vector/value_vector.cpp

@@ -2,6 +2,7 @@
 
 #include <numeric>
 
+#include "common/assert.h"
 #include "common/exception/runtime.h"
 #include "common/null_buffer.h"
 #include "common/serializer/deserializer.h"
@@ -26,6 +27,11 @@ ValueVector::ValueVector(LogicalType dataType, storage::MemoryManager* memoryMan
                                "Data type is expected to be resolved during binding.");
         // LCOV_EXCL_STOP
     }
+    // Strings too long to inline live in overflow blocks from the memory manager; nested types
+    // pass it to their child vectors, so the check covers strings at any depth.
+    DASSERT(
+        memoryManager != nullptr || (this->dataType.getPhysicalType() != PhysicalTypeID::STRING &&
+                                        this->dataType.getPhysicalType() != PhysicalTypeID::JSON));
     numBytesPerValue = getDataTypeSize(this->dataType);
     initializeValueBuffer();
     auxiliaryBuffer = AuxiliaryBufferFactory::getAuxiliaryBuffer(this->dataType, memoryManager);

sdist/ladybug-source/src/function/table/cache_column.cpp

@@ -6,6 +6,7 @@
 #include "function/table/bind_input.h"
 #include "function/table/simple_table_function.h"
 #include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
 #include "storage/local_cached_column.h"
 #include "storage/partition_storage_registry.h"
 #include "storage/storage_manager.h"
@@ -98,7 +99,8 @@ struct CacheArrayColumnLocalState final : TableFuncLocalState {
                           ->cast<storage::NodeTable>();
         dataChunk.insert(0, std::make_shared<ValueVector>(LogicalType::INTERNAL_ID()));
         dataChunk.insert(1,
-            std::make_shared<ValueVector>(table.getColumn(columnID).getDataType().copy()));
+            std::make_shared<ValueVector>(table.getColumn(columnID).getDataType().copy(),
+                storage::MemoryManager::Get(context)));
         std::vector<column_id_t> columnIDs;
         columnIDs.push_back(columnID);
         scanState =

sdist/ladybug-source/src/include/binder/ddl/property_definition.h

@@ -33,6 +33,7 @@ struct LBUG_API PropertyDefinition {
     std::string getName() const { return columnDefinition.name; }
     const common::LogicalType& getType() const { return columnDefinition.type; }
     std::string getDefaultExpressionName() const { return defaultExpr->getRawName(); }
+    bool hasDefault() const;
     void rename(const std::string& newName) { columnDefinition.name = newName; }
 
     void serialize(common::Serializer& serializer) const;

sdist/ladybug-source/src/include/catalog/catalog.h

@@ -190,9 +190,11 @@ public:
 
     // ----------------------------- Macro ----------------------------
 
-    // Check if macro entry exists.
+    // Macro names are stored uppercased (see bindCreateMacro). containsMacro and dropMacro
+    // uppercase the given name the same way; the catalog set itself only folds ASCII letters.
     bool containsMacro(const transaction::Transaction* transaction,
         const std::string& macroName) const;
+    // Takes the stored (uppercased) name.
     void addScalarMacroFunction(transaction::Transaction* transaction, std::string name,
         std::unique_ptr<function::ScalarMacroFunction> macro);
     ScalarMacroCatalogEntry* getScalarMacroCatalogEntry(const transaction::Transaction* transaction,
@@ -200,6 +202,7 @@ public:
     void dropMacroEntry(transaction::Transaction* transaction, const lbug::common::oid_t macroID);
     void dropMacroEntry(transaction::Transaction* transaction,
         const ScalarMacroCatalogEntry* entry);
+    // Takes the stored (uppercased) name.
     function::ScalarMacroFunction* getScalarMacroFunction(
         const transaction::Transaction* transaction, const std::string& name) const;
     std::vector<std::string> getMacroNames(const transaction::Transaction* transaction) const;

sdist/ladybug-source/src/include/common/constants.h

@@ -178,8 +178,8 @@ struct PlannerKnobs {
     // cost vs per-output-pair cost is ~1000:1, so seed cardinality dominates direction choice.
     static constexpr uint64_t RECURSIVE_EXTEND_SOURCE_COST = 1000;
     // Lookup join (per-tuple PK lookup instead of scan + hash join) pays off only when the
-    // outer side is small: each outer row costs one index probe/payload fetch, so cap the
-    // estimated outer cardinality this rewrite applies to (Q1's Person fetch: ~9 rows).
+    // outer side is small: each flat outer row costs one index probe/payload fetch, so cap the
+    // estimated number of flat outer rows this rewrite applies to (Q1's Person fetch: ~9 rows).
     static constexpr uint64_t LOOKUP_JOIN_MAX_OUTER_CARDINALITY = 1024;
     // Lookup join eligibility additionally requires the outer plan to be free of
     // unmodeled fan-out (no recursion/aggregation/etc.) and caps the extend chain

sdist/ladybug-source/src/include/optimizer/remove_unnecessary_join_optimizer.h

@@ -15,7 +15,8 @@ namespace optimizer {
  *   E(e) S(b)
  *    |
  *   S(a)
- * This optimizer prunes such redundant joins.
+ * This optimizer prunes such redundant joins. A join whose key an OPTIONAL MATCH can leave
+ * NULL is not redundant: it drops the rows with a NULL key, so it stays.
  */
 class RemoveUnnecessaryJoinOptimizer : public LogicalOperatorVisitor {
 public:

sdist/ladybug-source/src/include/planner/join_order/cost_model.h

@@ -22,11 +22,10 @@ public:
         const LogicalPlan& probe, const LogicalPlan& build);
     static uint64_t computeIntersectCost(const LogicalPlan& probePlan,
         const std::vector<LogicalPlan>& buildPlans);
-    // Lookup join: one PK probe/payload fetch per outer row on top of the outer plan.
+    // Lookup join: one PK probe/payload fetch per flat outer row on top of the outer plan.
     // Like the INL-join precedent, this only pays off for small outer cardinalities,
     // which callers enforce via PlannerKnobs::LOOKUP_JOIN_MAX_OUTER_CARDINALITY.
-    static uint64_t computeLookupJoinCost(const LogicalPlan& outerPlan,
-        cardinality_t outerCardinality);
+    static uint64_t computeLookupJoinCost(const LogicalPlan& outerPlan, cardinality_t numProbes);
 };
 
 } // namespace planner

sdist/ladybug-source/src/include/planner/operator/logical_hash_join.h

@@ -37,7 +37,7 @@ public:
     static binder::expression_vector getJoinNodeIDs(
         const std::vector<join_condition_t>& joinConditions);
 
-    std::vector<join_condition_t> getJoinConditions() const { return joinConditions; }
+    const std::vector<join_condition_t>& getJoinConditions() const { return joinConditions; }
     common::JoinType getJoinType() const { return joinType; }
     bool hasMark() const { return mark != nullptr; }
     std::shared_ptr<binder::Expression> getMark() const { return mark; }

sdist/ladybug-source/src/include/processor/operator/ddl/drop.h

@@ -42,7 +42,9 @@ private:
     void dropMacro(const main::ClientContext* context);
     void dropGraph(const main::ClientContext* context);
     void dropIndex(const main::ClientContext* context);
-    void handleMacroExistence(const main::ClientContext* context);
+    // Returns false if the macro is missing and IF EXISTS was given; throws if it is missing
+    // otherwise.
+    bool handleMacroExistence(const main::ClientContext* context);
     void dropRelGroup(const main::ClientContext* context);
 
 private:

sdist/ladybug-source/src/include/processor/operator/persistent/reader/parquet/parquet_reader.h

@@ -11,6 +11,10 @@
 #include "resizable_buffer.h"
 
 namespace lbug {
+namespace storage {
+class MemoryManager;
+} // namespace storage
+
 namespace processor {
 
 struct ParquetReaderPrefetchConfig {
@@ -58,6 +62,7 @@ public:
     const common::LogicalType& getColumnType(uint32_t idx) const { return columnTypes[idx]; }
 
     lbug_parquet::format::FileMetaData* getMetadata() const { return metadata.get(); }
+    storage::MemoryManager* getMemoryManager() const;
 
 private:
     std::unique_ptr<lbug_apache::thrift::protocol::TProtocol> createThriftProtocol(

sdist/ladybug-source/src/include/processor/result/factorized_table.h

@@ -45,6 +45,9 @@ public:
         }
         return data;
     }
+    // Gives back the claimed bytes past the first numBytes. They may still hold old tuples, so
+    // they are zeroed again if claimed later.
+    void truncate(uint64_t numBytes);
     void resetNumTuplesAndFreeSize();
     void resetToZero();
 

sdist/ladybug-source/src/include/storage/table/csr_chunked_node_group.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <algorithm>
+#include <utility>
+#include <vector>
 
 #include "storage/enums/residency_state.h"
 #include "storage/table/chunked_node_group.h"
@@ -62,6 +64,18 @@ struct LBUG_API InMemChunkedCSRHeader {
     bool randomLookup = false;
     // Logical header index stored at position 0. A full-group header uses 0.
     common::offset_t indexBase = 0;
+    // Scan-private cache: a full on-disk header, or the [start, end) index ranges last decoded
+    // for a bound-node batch. Nodes at or past headerLimit, the on-disk header size, have no
+    // lists and read as empty. Checkpoint headers leave these at the defaults.
+    bool wholeHeader = false;
+    common::offset_t headerLimit = 0;
+    std::vector<std::pair<common::offset_t, common::offset_t>> decodedRanges;
+
+    // Scans read the header only for nodes of the current bound-node batch (plus the group's end
+    // after a whole-header decode), and CSRNodeGroup::initializeScanState covers each batch by
+    // the decode it runs for it or by checking these ranges. The getters assert it; an uncovered
+    // read returns stale bytes from a gap, or the last entry at or past numValues.
+    bool coversNode(common::offset_t nodeOffset) const;
 
     InMemChunkedCSRHeader(MemoryManager& memoryManager, bool enableCompression, uint64_t capacity);
     InMemChunkedCSRHeader(std::unique_ptr<ColumnChunkData> offset,

sdist/ladybug-source/src/include/storage/table/csr_node_group.h

@@ -145,7 +145,8 @@ struct CSRNodeGroupScanState final : NodeGroupScanState {
     // The number of rows (i.e., rels) that have been scanned so far in current node group.
     common::row_idx_t numCachedRows;
     common::row_idx_t nextCachedRowToScan;
-    // Entries and spans decoded so far for the current node group's header.
+    // Entries decoded and partial decodes (spans or cluster sets) so far for the current node
+    // group's header.
     common::offset_t numHeaderEntriesDecoded = 0;
     common::idx_t numHeaderSpansDecoded = 0;
 
@@ -178,6 +179,12 @@ struct CSRNodeGroupScanState final : NodeGroupScanState {
     // node vector's chunk state, which holds the served parents in its selection vector. See
     // docs/multi_parent_lifetime.md.
     bool tryScanCachedTuplesPacked(RelTableScanState& tableScanState);
+
+    // Copies rows into the cache starting in the current bound node's list and running through
+    // the next lists up to a gap (see refillEnd), or moves past that node when its list is empty
+    // or already scanned. False once the batch has no rows left.
+    bool refillCache(const transaction::Transaction* transaction,
+        RelTableScanState& tableScanState);
 };
 
 struct CSRNodeGroupCheckpointState final : NodeGroupCheckpointState {
@@ -264,6 +271,7 @@ public:
         return NodeGroup::hasRows() ||
                (persistentChunkGroup && persistentChunkGroup->getNumRows() != 0);
     }
+    void rollbackInsert(common::row_idx_t startRow) override;
 
     ChunkedNodeGroup* getPersistentChunkedGroup() const { return persistentChunkGroup.get(); }
     const CSRIndex* getCSRIndex() const { return csrIndex.get(); }
@@ -286,8 +294,8 @@ private:
     static void initScanForCommittedInMem(RelTableScanState& relScanState,
         CSRNodeGroupScanState& nodeGroupScanState);
 
-    void updateCSRIndex(common::offset_t boundNodeOffsetInGroup, common::row_idx_t startRow,
-        common::length_t length) const;
+    static void updateCSRIndex(CSRIndex& index, common::offset_t boundNodeOffsetInGroup,
+        common::row_idx_t startRow, common::length_t length);
 
     NodeGroupScanResult scanCommittedPersistent(const transaction::Transaction* transaction,
         RelTableScanState& tableState, CSRNodeGroupScanState& nodeGroupScanState) const;
@@ -313,7 +321,7 @@ private:
         const transaction::Transaction* transaction, CSRNodeGroupScanPin& pin) const;
     void pruneRetainedSnapshotsNoLock(const std::vector<common::transaction_t>& openStartTSs);
     bool needsRetainedSnapshotNoLock(const std::vector<common::transaction_t>& openStartTSs) const;
-    void ensureCSRIndex();
+    CSRIndex& getCSRIndexForAppend(const common::UniqLock& lock);
 
     void checkpointInMemOnly(const common::UniqLock& lock, NodeGroupCheckpointState& state);
     void checkpointInMemAndOnDisk(const common::UniqLock& lock, NodeGroupCheckpointState& state);

sdist/ladybug-source/src/include/storage/table/node_group.h

@@ -25,7 +25,8 @@ struct NodeGroupScanState {
     // Index of committed but not yet checkpointed chunked group to scan.
     common::idx_t chunkedGroupIdx = 0;
     // Batches to scan without filtering on predicates first, after a batch where that kept too
-    // many rows, and the length of the last such pause.
+    // many rows, and the length of the last such pause. Reset on entering a node group, not a
+    // chunked group: an in-memory chunked group holds a single batch.
     uint8_t numBatchesWithoutEarlyFilter = 0;
     uint8_t earlyFilterBackoff = 0;
     common::row_idx_t nextRowToScan = 0;
@@ -37,6 +38,11 @@ struct NodeGroupScanState {
     explicit NodeGroupScanState() {}
     explicit NodeGroupScanState(common::idx_t numChunks) { chunkStates.resize(numChunks); }
 
+    void resetEarlyFilterBackoff() {
+        numBatchesWithoutEarlyFilter = 0;
+        earlyFilterBackoff = 0;
+    }
+
     virtual ~NodeGroupScanState() = default;
     DELETE_COPY_DEFAULT_MOVE(NodeGroupScanState);
 
@@ -189,7 +195,7 @@ public:
 
     void applyFuncToChunkedGroups(version_record_handler_op_t func, common::row_idx_t startRow,
         common::row_idx_t numRows, common::transaction_t commitTS) const;
-    void rollbackInsert(common::row_idx_t startRow);
+    virtual void rollbackInsert(common::row_idx_t startRow);
     void reclaimStorage(PageAllocator& pageAllocator) const;
     virtual void reclaimStorage(PageAllocator& pageAllocator, const common::UniqLock& lock) const;
 

sdist/ladybug-source/src/include/storage/table/node_table.h

@@ -274,9 +274,8 @@ private:
         std::optional<uint64_t> queryID = std::nullopt) const;
 
 private:
-    // Protects the `columns` vector from concurrent access during checkpoint
-    // column vacuum. Readers/writers take shared_lock; checkpoint takes
-    // unique_lock during the brief column swap.
+    // Guards the `columns` vector: checkpoint's column vacuum swaps it and ALTER TABLE ADD grows
+    // it under a unique lock; readers take a shared lock.
     mutable std::shared_mutex schemaMtx;
     std::vector<std::unique_ptr<Column>> columns;
     std::unique_ptr<NodeGroupCollection> nodeGroups;

sdist/ladybug-source/src/include/storage/table/rel_table_data.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cmath>
+#include <shared_mutex>
 
 #include "common/enums/rel_direction.h"
 #include "common/enums/rel_multiplicity.h"
@@ -71,12 +72,20 @@ public:
     bool checkIfNodeHasRels(transaction::Transaction* transaction,
         common::ValueVector* srcNodeIDVector) const;
 
-    Column* getNbrIDColumn() const { return columns[NBR_ID_COLUMN_ID].get(); }
+    Column* getNbrIDColumn() const { return getColumn(NBR_ID_COLUMN_ID); }
     Column* getCSROffsetColumn() const { return csrHeaderColumns.offset.get(); }
     Column* getCSRLengthColumn() const { return csrHeaderColumns.length.get(); }
-    common::column_id_t getNumColumns() const { return columns.size(); }
-    Column* getColumn(common::column_id_t columnID) const { return columns[columnID].get(); }
+    common::column_id_t getNumColumns() const {
+        std::shared_lock lck{columnsMtx};
+        return columns.size();
+    }
+    Column* getColumn(common::column_id_t columnID) const {
+        std::shared_lock lck{columnsMtx};
+        DASSERT(columnID < columns.size());
+        return columns[columnID].get();
+    }
     std::vector<const Column*> getColumns() const {
+        std::shared_lock lck{columnsMtx};
         std::vector<const Column*> result;
         result.reserve(columns.size());
         for (const auto& column : columns) {
@@ -139,6 +148,7 @@ private:
     }
 
     std::vector<common::LogicalType> getColumnTypes() const {
+        std::shared_lock lck{columnsMtx};
         std::vector<common::LogicalType> types;
         types.reserve(columns.size());
         for (const auto& column : columns) {
@@ -161,6 +171,10 @@ private:
     std::unique_ptr<NodeGroupCollection> nodeGroups;
 
     CSRHeaderColumns csrHeaderColumns;
+    // Guards the `columns` vector, as NodeTable::schemaMtx guards the node table's: checkpoint
+    // swaps it and ALTER TABLE ADD grows it under a unique lock while other connections' scans
+    // read it under a shared lock. A Column* stays valid for every column a checkpoint keeps.
+    mutable std::shared_mutex columnsMtx;
     std::vector<std::unique_ptr<Column>> columns;
 
     PersistentVersionRecordHandler persistentVersionRecordHandler;

sdist/ladybug-source/src/include/storage/table/version_info.h

@@ -32,6 +32,7 @@ public:
     void clearVectorInfo(common::idx_t vectorIdx);
 
     bool hasDeletions() const;
+    // Rows whose delete the transaction can see. Insert stamps are not checked.
     common::row_idx_t getNumDeletions(const transaction::Transaction* transaction,
         common::row_idx_t startRow, common::length_t numRows) const;
     common::row_idx_t getNumVisibleRows(const transaction::Transaction* transaction,
@@ -39,7 +40,8 @@ public:
     // Rows whose insert the transaction can see, deleted or not.
     common::row_idx_t getNumInsertedRows(const transaction::Transaction* transaction,
         common::row_idx_t startRow, common::length_t numRows) const;
-    // Appends the rows in [startRow, startRow + numRows) whose delete the transaction can see.
+    // Appends to `rows` the rows in [startRow, startRow + numRows) whose delete the transaction can
+    // see, as row indices in the group, not offsets from startRow. Insert stamps are not checked.
     void getDeletedRows(const transaction::Transaction* transaction, common::row_idx_t startRow,
         common::length_t numRows, std::vector<common::row_idx_t>& rows) const;
     bool hasInsertions() const;