kuzu 0.11.1.dev6__tar.gz → 0.11.1.dev8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdist/PKG-INFO +1 -1
- sdist/kuzu-source/CMakeLists.txt +1 -1
- sdist/kuzu-source/src/c_api/value.cpp +15 -0
- sdist/kuzu-source/src/graph/on_disk_graph.cpp +1 -1
- sdist/kuzu-source/src/include/c_api/kuzu.h +7 -0
- sdist/kuzu-source/src/include/storage/local_storage/local_node_table.h +1 -2
- sdist/kuzu-source/src/include/storage/local_storage/local_rel_table.h +3 -3
- sdist/kuzu-source/src/include/storage/local_storage/local_table.h +1 -2
- sdist/kuzu-source/src/include/storage/table/chunked_node_group.h +4 -4
- sdist/kuzu-source/src/include/storage/table/column_chunk.h +6 -6
- sdist/kuzu-source/src/include/storage/table/csr_chunked_node_group.h +3 -2
- sdist/kuzu-source/src/include/storage/table/csr_node_group.h +9 -10
- sdist/kuzu-source/src/include/storage/table/node_group.h +17 -17
- sdist/kuzu-source/src/include/storage/table/node_group_collection.h +5 -4
- sdist/kuzu-source/src/include/storage/table/node_table.h +1 -1
- sdist/kuzu-source/src/include/storage/table/rel_table_data.h +5 -6
- sdist/kuzu-source/src/include/transaction/transaction.h +0 -1
- sdist/kuzu-source/src/processor/operator/persistent/rel_batch_insert.cpp +1 -1
- sdist/kuzu-source/src/storage/local_storage/local_node_table.cpp +4 -4
- sdist/kuzu-source/src/storage/local_storage/local_rel_table.cpp +9 -8
- sdist/kuzu-source/src/storage/local_storage/local_storage.cpp +3 -3
- sdist/kuzu-source/src/storage/table/chunked_node_group.cpp +8 -10
- sdist/kuzu-source/src/storage/table/column_chunk.cpp +15 -17
- sdist/kuzu-source/src/storage/table/csr_chunked_node_group.cpp +6 -5
- sdist/kuzu-source/src/storage/table/csr_node_group.cpp +15 -15
- sdist/kuzu-source/src/storage/table/node_group.cpp +12 -15
- sdist/kuzu-source/src/storage/table/node_group_collection.cpp +22 -19
- sdist/kuzu-source/src/storage/table/node_table.cpp +12 -15
- sdist/kuzu-source/src/storage/table/rel_table.cpp +8 -12
- sdist/kuzu-source/src/storage/table/rel_table_data.cpp +23 -29
- sdist/kuzu-source/tools/nodejs_api/package.json +2 -2
- sdist/kuzu-source/tools/nodejs_api/src_js/kuzu.d.ts +113 -107
- sdist/kuzu.egg-info/PKG-INFO +1 -1
- sdist/pyproject.toml +1 -1
sdist/PKG-INFO
CHANGED
sdist/kuzu-source/CMakeLists.txt
CHANGED
@@ -856,6 +856,21 @@ kuzu_state kuzu_node_val_to_string(kuzu_value* node_val, char** out_result) {
|
|
856
856
|
return KuzuSuccess;
|
857
857
|
}
|
858
858
|
|
859
|
+
kuzu_state kuzu_rel_val_get_id_val(kuzu_value* rel_val, kuzu_value* out_value) {
|
860
|
+
auto logical_type_id = static_cast<Value*>(rel_val->_value)->getDataType().getLogicalTypeID();
|
861
|
+
if (logical_type_id != LogicalTypeID::REL) {
|
862
|
+
return KuzuError;
|
863
|
+
}
|
864
|
+
try {
|
865
|
+
auto id_val = RelVal::getIDVal(static_cast<Value*>(rel_val->_value));
|
866
|
+
out_value->_value = id_val;
|
867
|
+
out_value->_is_owned_by_cpp = true;
|
868
|
+
} catch (Exception& e) {
|
869
|
+
return KuzuError;
|
870
|
+
}
|
871
|
+
return KuzuSuccess;
|
872
|
+
}
|
873
|
+
|
859
874
|
kuzu_state kuzu_rel_val_get_src_id_val(kuzu_value* rel_val, kuzu_value* out_value) {
|
860
875
|
auto logical_type_id = static_cast<Value*>(rel_val->_value)->getDataType().getLogicalTypeID();
|
861
876
|
if (logical_type_id != LogicalTypeID::REL) {
|
@@ -339,7 +339,7 @@ bool OnDiskGraphVertexScanState::next() {
|
|
339
339
|
numNodesToScan = std::min(endOffset - currentOffset, DEFAULT_VECTOR_CAPACITY);
|
340
340
|
auto result = tableScanState->scanNext(context.getTransaction(), currentOffset, numNodesToScan);
|
341
341
|
currentOffset += result.numRows;
|
342
|
-
return result !=
|
342
|
+
return result != NODE_GROUP_SCAN_EMPTY_RESULT;
|
343
343
|
}
|
344
344
|
|
345
345
|
} // namespace graph
|
@@ -1366,6 +1366,13 @@ KUZU_C_API kuzu_state kuzu_node_val_get_property_value_at(kuzu_value* node_val,
|
|
1366
1366
|
* @return The state indicating the success or failure of the operation.
|
1367
1367
|
*/
|
1368
1368
|
KUZU_C_API kuzu_state kuzu_node_val_to_string(kuzu_value* node_val, char** out_result);
|
1369
|
+
/**
|
1370
|
+
* @brief Returns the internal id value of the rel value as a kuzu value.
|
1371
|
+
* @param rel_val The rel value to return.
|
1372
|
+
* @param[out] out_value The output parameter that will hold the internal id value.
|
1373
|
+
* @return The state indicating the success or failure of the operation.
|
1374
|
+
*/
|
1375
|
+
KUZU_C_API kuzu_state kuzu_rel_val_get_id_val(kuzu_value* rel_val, kuzu_value* out_value);
|
1369
1376
|
/**
|
1370
1377
|
* @brief Returns the internal id value of the source node of the given rel value as a kuzu value.
|
1371
1378
|
* @param rel_val The rel value to return.
|
@@ -19,8 +19,7 @@ public:
|
|
19
19
|
bool insert(transaction::Transaction* transaction, TableInsertState& insertState) override;
|
20
20
|
bool update(transaction::Transaction* transaction, TableUpdateState& updateState) override;
|
21
21
|
bool delete_(transaction::Transaction* transaction, TableDeleteState& deleteState) override;
|
22
|
-
bool addColumn(
|
23
|
-
TableAddColumnState& addColumnState) override;
|
22
|
+
bool addColumn(TableAddColumnState& addColumnState) override;
|
24
23
|
uint64_t getEstimatedMemUsage() override;
|
25
24
|
|
26
25
|
common::offset_t validateUniquenessConstraint(const transaction::Transaction* transaction,
|
@@ -32,14 +32,14 @@ struct DirectedCSRIndex {
|
|
32
32
|
|
33
33
|
class LocalRelTable final : public LocalTable {
|
34
34
|
public:
|
35
|
-
LocalRelTable(const catalog::TableCatalogEntry* tableEntry, const Table& table
|
35
|
+
LocalRelTable(const catalog::TableCatalogEntry* tableEntry, const Table& table,
|
36
|
+
MemoryManager& mm);
|
36
37
|
DELETE_COPY_AND_MOVE(LocalRelTable);
|
37
38
|
|
38
39
|
bool insert(transaction::Transaction* transaction, TableInsertState& state) override;
|
39
40
|
bool update(transaction::Transaction* transaction, TableUpdateState& state) override;
|
40
41
|
bool delete_(transaction::Transaction* transaction, TableDeleteState& state) override;
|
41
|
-
bool addColumn(
|
42
|
-
TableAddColumnState& addColumnState) override;
|
42
|
+
bool addColumn(TableAddColumnState& addColumnState) override;
|
43
43
|
uint64_t getEstimatedMemUsage() override;
|
44
44
|
|
45
45
|
bool checkIfNodeHasRels(common::ValueVector* srcNodeIDVector,
|
@@ -22,8 +22,7 @@ public:
|
|
22
22
|
virtual bool insert(transaction::Transaction* transaction, TableInsertState& insertState) = 0;
|
23
23
|
virtual bool update(transaction::Transaction* transaction, TableUpdateState& updateState) = 0;
|
24
24
|
virtual bool delete_(transaction::Transaction* transaction, TableDeleteState& deleteState) = 0;
|
25
|
-
virtual bool addColumn(
|
26
|
-
TableAddColumnState& addColumnState) = 0;
|
25
|
+
virtual bool addColumn(TableAddColumnState& addColumnState) = 0;
|
27
26
|
virtual void clear(MemoryManager& mm) = 0;
|
28
27
|
virtual common::TableType getTableType() const = 0;
|
29
28
|
virtual uint64_t getEstimatedMemUsage() = 0;
|
@@ -121,9 +121,8 @@ public:
|
|
121
121
|
|
122
122
|
bool delete_(const transaction::Transaction* transaction, common::row_idx_t rowIdxInChunk);
|
123
123
|
|
124
|
-
void addColumn(const
|
125
|
-
|
126
|
-
PageAllocator* pageAllocator, ColumnStats* newColumnStats);
|
124
|
+
void addColumn(MemoryManager& mm, const TableAddColumnState& addColumnState,
|
125
|
+
bool enableCompression, PageAllocator* pageAllocator, ColumnStats* newColumnStats);
|
127
126
|
|
128
127
|
bool isDeleted(const transaction::Transaction* transaction, common::row_idx_t rowInChunk) const;
|
129
128
|
bool isInserted(const transaction::Transaction* transaction,
|
@@ -144,7 +143,8 @@ public:
|
|
144
143
|
}
|
145
144
|
|
146
145
|
virtual std::unique_ptr<ChunkedNodeGroup> flushAsNewChunkedNodeGroup(
|
147
|
-
transaction::Transaction* transaction,
|
146
|
+
transaction::Transaction* transaction, MemoryManager& mm,
|
147
|
+
PageAllocator& pageAllocator) const;
|
148
148
|
virtual void flush(PageAllocator& pageAllocator);
|
149
149
|
|
150
150
|
void commitInsert(common::row_idx_t startRow, common::row_idx_t numRowsToCommit,
|
@@ -38,11 +38,11 @@ struct ColumnCheckpointState {
|
|
38
38
|
|
39
39
|
class ColumnChunk {
|
40
40
|
public:
|
41
|
-
ColumnChunk(MemoryManager&
|
41
|
+
ColumnChunk(MemoryManager& mm, common::LogicalType&& dataType, uint64_t capacity,
|
42
42
|
bool enableCompression, ResidencyState residencyState, bool initializeToZero = true);
|
43
|
-
ColumnChunk(MemoryManager&
|
44
|
-
|
45
|
-
ColumnChunk(bool enableCompression, std::unique_ptr<ColumnChunkData> data);
|
43
|
+
ColumnChunk(MemoryManager& mm, common::LogicalType&& dataType, bool enableCompression,
|
44
|
+
ColumnChunkMetadata metadata);
|
45
|
+
ColumnChunk(MemoryManager& mm, bool enableCompression, std::unique_ptr<ColumnChunkData> data);
|
46
46
|
|
47
47
|
void initializeScanState(ChunkState& state, const Column* column) const;
|
48
48
|
void scan(const transaction::Transaction* transaction, const ChunkState& state,
|
@@ -61,8 +61,7 @@ public:
|
|
61
61
|
return getResidencyState() == ResidencyState::ON_DISK ? 0 : data->getEstimatedMemoryUsage();
|
62
62
|
}
|
63
63
|
void serialize(common::Serializer& serializer) const;
|
64
|
-
static std::unique_ptr<ColumnChunk> deserialize(MemoryManager&
|
65
|
-
common::Deserializer& deSer);
|
64
|
+
static std::unique_ptr<ColumnChunk> deserialize(MemoryManager& mm, common::Deserializer& deSer);
|
66
65
|
|
67
66
|
uint64_t getNumValues() const { return data->getNumValues(); }
|
68
67
|
void setNumValues(const uint64_t numValues) const { data->setNumValues(numValues); }
|
@@ -110,6 +109,7 @@ private:
|
|
110
109
|
common::row_idx_t numRows) const;
|
111
110
|
|
112
111
|
private:
|
112
|
+
MemoryManager& mm;
|
113
113
|
// TODO(Guodong): This field should be removed. Ideally it shouldn't be cached anywhere in
|
114
114
|
// storage structures, instead should be fed into functions needed from ClientContext dbConfig.
|
115
115
|
bool enableCompression;
|
@@ -66,7 +66,7 @@ struct KUZU_API ChunkedCSRHeader {
|
|
66
66
|
common::offset_t getStartCSROffset(common::offset_t nodeOffset) const;
|
67
67
|
common::offset_t getEndCSROffset(common::offset_t nodeOffset) const;
|
68
68
|
common::length_t getCSRLength(common::offset_t nodeOffset) const;
|
69
|
-
common::length_t getGapSize(common::length_t
|
69
|
+
common::length_t getGapSize(common::length_t nodeOffset) const;
|
70
70
|
|
71
71
|
bool sanityCheck() const;
|
72
72
|
void copyFrom(const ChunkedCSRHeader& other) const;
|
@@ -127,7 +127,8 @@ public:
|
|
127
127
|
void scanCSRHeader(MemoryManager& memoryManager, CSRNodeGroupCheckpointState& csrState) const;
|
128
128
|
|
129
129
|
std::unique_ptr<ChunkedNodeGroup> flushAsNewChunkedNodeGroup(
|
130
|
-
transaction::Transaction* transaction,
|
130
|
+
transaction::Transaction* transaction, MemoryManager& mm,
|
131
|
+
PageAllocator& pageAllocator) const override;
|
131
132
|
|
132
133
|
void flush(PageAllocator& pageAllocator) override;
|
133
134
|
void reclaimStorage(PageAllocator& pageAllocator) const override;
|
@@ -169,18 +169,17 @@ static constexpr common::column_id_t REL_ID_COLUMN_ID = 1;
|
|
169
169
|
// Transient data are organized similar to normal node groups. Tuples are always appended to the end
|
170
170
|
// of `chunkedGroups`. We keep an extra csrIndex to track the vector of row indices for each bound
|
171
171
|
// node.
|
172
|
-
struct RelTableScanState;
|
173
172
|
class CSRNodeGroup final : public NodeGroup {
|
174
173
|
public:
|
175
174
|
static constexpr PackedCSRInfo DEFAULT_PACKED_CSR_INFO{};
|
176
175
|
|
177
|
-
CSRNodeGroup(const common::node_group_idx_t nodeGroupIdx,
|
178
|
-
std::vector<common::LogicalType> dataTypes)
|
179
|
-
: NodeGroup{nodeGroupIdx, enableCompression, std::move(dataTypes),
|
180
|
-
NodeGroupDataFormat::CSR} {}
|
181
|
-
CSRNodeGroup(const common::node_group_idx_t nodeGroupIdx,
|
182
|
-
std::unique_ptr<ChunkedNodeGroup> chunkedNodeGroup)
|
183
|
-
: NodeGroup{nodeGroupIdx, enableCompression, common::INVALID_OFFSET,
|
176
|
+
CSRNodeGroup(MemoryManager& mm, const common::node_group_idx_t nodeGroupIdx,
|
177
|
+
const bool enableCompression, std::vector<common::LogicalType> dataTypes)
|
178
|
+
: NodeGroup{mm, nodeGroupIdx, enableCompression, std::move(dataTypes),
|
179
|
+
common::INVALID_OFFSET, NodeGroupDataFormat::CSR} {}
|
180
|
+
CSRNodeGroup(MemoryManager& mm, const common::node_group_idx_t nodeGroupIdx,
|
181
|
+
const bool enableCompression, std::unique_ptr<ChunkedNodeGroup> chunkedNodeGroup)
|
182
|
+
: NodeGroup{mm, nodeGroupIdx, enableCompression, common::INVALID_OFFSET,
|
184
183
|
NodeGroupDataFormat::CSR},
|
185
184
|
persistentChunkGroup{std::move(chunkedNodeGroup)} {
|
186
185
|
for (auto i = 0u; i < persistentChunkGroup->getNumColumns(); i++) {
|
@@ -206,8 +205,8 @@ public:
|
|
206
205
|
bool delete_(const transaction::Transaction* transaction, CSRNodeGroupScanSource source,
|
207
206
|
common::row_idx_t rowIdxInGroup);
|
208
207
|
|
209
|
-
void addColumn(
|
210
|
-
|
208
|
+
void addColumn(TableAddColumnState& addColumnState, PageAllocator* pageAllocator,
|
209
|
+
ColumnStats* newColumnStats) override;
|
211
210
|
|
212
211
|
void checkpoint(MemoryManager& memoryManager, NodeGroupCheckpointState& state) override;
|
213
212
|
void reclaimStorage(PageAllocator& pageAllocator, const common::UniqLock& lock) const override;
|
@@ -79,22 +79,22 @@ struct NodeGroupScanResult {
|
|
79
79
|
}
|
80
80
|
};
|
81
81
|
|
82
|
-
static auto
|
82
|
+
static auto NODE_GROUP_SCAN_EMPTY_RESULT = NodeGroupScanResult{};
|
83
83
|
|
84
84
|
struct TableScanState;
|
85
85
|
class NodeGroup {
|
86
86
|
public:
|
87
|
-
NodeGroup(const common::node_group_idx_t nodeGroupIdx,
|
88
|
-
std::vector<common::LogicalType> dataTypes,
|
87
|
+
NodeGroup(MemoryManager& mm, const common::node_group_idx_t nodeGroupIdx,
|
88
|
+
const bool enableCompression, std::vector<common::LogicalType> dataTypes,
|
89
89
|
common::row_idx_t capacity = common::StorageConfig::NODE_GROUP_SIZE,
|
90
90
|
NodeGroupDataFormat format = NodeGroupDataFormat::REGULAR)
|
91
|
-
: nodeGroupIdx{nodeGroupIdx}, format{format}, enableCompression{enableCompression},
|
91
|
+
: mm{mm}, nodeGroupIdx{nodeGroupIdx}, format{format}, enableCompression{enableCompression},
|
92
92
|
numRows{0}, nextRowToAppend{0}, capacity{capacity}, dataTypes{std::move(dataTypes)} {}
|
93
|
-
NodeGroup(const common::node_group_idx_t nodeGroupIdx,
|
94
|
-
std::unique_ptr<ChunkedNodeGroup> chunkedNodeGroup,
|
93
|
+
NodeGroup(MemoryManager& mm, const common::node_group_idx_t nodeGroupIdx,
|
94
|
+
const bool enableCompression, std::unique_ptr<ChunkedNodeGroup> chunkedNodeGroup,
|
95
95
|
common::row_idx_t capacity = common::StorageConfig::NODE_GROUP_SIZE,
|
96
96
|
NodeGroupDataFormat format = NodeGroupDataFormat::REGULAR)
|
97
|
-
: nodeGroupIdx{nodeGroupIdx}, format{format}, enableCompression{enableCompression},
|
97
|
+
: mm{mm}, nodeGroupIdx{nodeGroupIdx}, format{format}, enableCompression{enableCompression},
|
98
98
|
numRows{chunkedNodeGroup->getStartRowIdx() + chunkedNodeGroup->getNumRows()},
|
99
99
|
nextRowToAppend{numRows}, capacity{capacity} {
|
100
100
|
for (auto i = 0u; i < chunkedNodeGroup->getNumColumns(); i++) {
|
@@ -103,9 +103,9 @@ public:
|
|
103
103
|
const auto lock = chunkedGroups.lock();
|
104
104
|
chunkedGroups.appendGroup(lock, std::move(chunkedNodeGroup));
|
105
105
|
}
|
106
|
-
NodeGroup(const common::node_group_idx_t nodeGroupIdx,
|
107
|
-
common::row_idx_t capacity, NodeGroupDataFormat format)
|
108
|
-
: nodeGroupIdx{nodeGroupIdx}, format{format}, enableCompression{enableCompression},
|
106
|
+
NodeGroup(MemoryManager& mm, const common::node_group_idx_t nodeGroupIdx,
|
107
|
+
const bool enableCompression, common::row_idx_t capacity, NodeGroupDataFormat format)
|
108
|
+
: mm{mm}, nodeGroupIdx{nodeGroupIdx}, format{format}, enableCompression{enableCompression},
|
109
109
|
numRows{0}, nextRowToAppend{0}, capacity{capacity} {}
|
110
110
|
virtual ~NodeGroup() = default;
|
111
111
|
|
@@ -157,8 +157,7 @@ public:
|
|
157
157
|
bool delete_(const transaction::Transaction* transaction, common::row_idx_t rowIdxInGroup);
|
158
158
|
|
159
159
|
bool hasDeletions(const transaction::Transaction* transaction) const;
|
160
|
-
virtual void addColumn(
|
161
|
-
TableAddColumnState& addColumnState, PageAllocator* pageAllocator,
|
160
|
+
virtual void addColumn(TableAddColumnState& addColumnState, PageAllocator* pageAllocator,
|
162
161
|
ColumnStats* newColumnStats);
|
163
162
|
|
164
163
|
void applyFuncToChunkedGroups(version_record_handler_op_t func, common::row_idx_t startRow,
|
@@ -172,8 +171,8 @@ public:
|
|
172
171
|
uint64_t getEstimatedMemoryUsage() const;
|
173
172
|
|
174
173
|
virtual void serialize(common::Serializer& serializer);
|
175
|
-
static std::unique_ptr<NodeGroup> deserialize(MemoryManager&
|
176
|
-
|
174
|
+
static std::unique_ptr<NodeGroup> deserialize(MemoryManager& mm, common::Deserializer& deSer,
|
175
|
+
const std::vector<common::LogicalType>& columnTypes);
|
177
176
|
|
178
177
|
common::node_group_idx_t getNumChunkedGroups() const {
|
179
178
|
const auto lock = chunkedGroups.lock();
|
@@ -188,9 +187,9 @@ public:
|
|
188
187
|
TARGET& cast() {
|
189
188
|
return common::ku_dynamic_cast<TARGET&>(*this);
|
190
189
|
}
|
191
|
-
template<class
|
192
|
-
const
|
193
|
-
return common::ku_dynamic_cast<const
|
190
|
+
template<class TARGET>
|
191
|
+
const TARGET& cast() const {
|
192
|
+
return common::ku_dynamic_cast<const TARGET&>(*this);
|
194
193
|
}
|
195
194
|
|
196
195
|
bool isVisible(const transaction::Transaction* transaction,
|
@@ -240,6 +239,7 @@ private:
|
|
240
239
|
common::row_idx_t getStartRowIdxInGroup(const common::UniqLock& lock) const;
|
241
240
|
|
242
241
|
protected:
|
242
|
+
MemoryManager& mm;
|
243
243
|
common::node_group_idx_t nodeGroupIdx;
|
244
244
|
NodeGroupDataFormat format;
|
245
245
|
bool enableCompression;
|
@@ -13,8 +13,8 @@ class MemoryManager;
|
|
13
13
|
|
14
14
|
class NodeGroupCollection {
|
15
15
|
public:
|
16
|
-
NodeGroupCollection(const std::vector<common::LogicalType>& types,
|
17
|
-
ResidencyState residency = ResidencyState::IN_MEMORY,
|
16
|
+
NodeGroupCollection(MemoryManager& mm, const std::vector<common::LogicalType>& types,
|
17
|
+
bool enableCompression, ResidencyState residency = ResidencyState::IN_MEMORY,
|
18
18
|
const VersionRecordHandler* versionRecordHandler = nullptr);
|
19
19
|
|
20
20
|
void append(const transaction::Transaction* transaction,
|
@@ -73,8 +73,7 @@ public:
|
|
73
73
|
|
74
74
|
common::column_id_t getNumColumns() const { return types.size(); }
|
75
75
|
|
76
|
-
void addColumn(
|
77
|
-
PageAllocator* pageAllocator = nullptr);
|
76
|
+
void addColumn(TableAddColumnState& addColumnState, PageAllocator* pageAllocator = nullptr);
|
78
77
|
|
79
78
|
uint64_t getEstimatedMemoryUsage() const;
|
80
79
|
|
@@ -111,6 +110,8 @@ private:
|
|
111
110
|
void pushInsertInfo(const transaction::Transaction* transaction, const NodeGroup* nodeGroup,
|
112
111
|
common::row_idx_t numRows);
|
113
112
|
|
113
|
+
private:
|
114
|
+
MemoryManager& mm;
|
114
115
|
bool enableCompression;
|
115
116
|
// Num rows in the collection regardless of deletions.
|
116
117
|
std::atomic<common::row_idx_t> numTotalRows;
|
@@ -103,7 +103,7 @@ class StorageManager;
|
|
103
103
|
class KUZU_API NodeTable final : public Table {
|
104
104
|
public:
|
105
105
|
NodeTable(const StorageManager* storageManager,
|
106
|
-
const catalog::NodeTableCatalogEntry* nodeTableEntry, MemoryManager*
|
106
|
+
const catalog::NodeTableCatalogEntry* nodeTableEntry, MemoryManager* mm);
|
107
107
|
|
108
108
|
common::row_idx_t getNumTotalRows(const transaction::Transaction* transaction) override;
|
109
109
|
|
@@ -16,6 +16,7 @@ namespace transaction {
|
|
16
16
|
class Transaction;
|
17
17
|
}
|
18
18
|
namespace storage {
|
19
|
+
class Table;
|
19
20
|
class MemoryManager;
|
20
21
|
class RelTableData;
|
21
22
|
|
@@ -55,7 +56,7 @@ private:
|
|
55
56
|
class RelTableData {
|
56
57
|
public:
|
57
58
|
RelTableData(FileHandle* dataFH, MemoryManager* mm, ShadowFile* shadowFile,
|
58
|
-
const catalog::RelGroupCatalogEntry& relGroupEntry,
|
59
|
+
const catalog::RelGroupCatalogEntry& relGroupEntry, Table& table,
|
59
60
|
common::RelDataDirection direction, common::table_id_t nbrTableID, bool enableCompression);
|
60
61
|
|
61
62
|
bool update(transaction::Transaction* transaction, common::ValueVector& boundNodeIDVector,
|
@@ -63,8 +64,7 @@ public:
|
|
63
64
|
const common::ValueVector& dataVector) const;
|
64
65
|
bool delete_(transaction::Transaction* transaction, common::ValueVector& boundNodeIDVector,
|
65
66
|
const common::ValueVector& relIDVector);
|
66
|
-
void addColumn(
|
67
|
-
PageAllocator& pageAllocator);
|
67
|
+
void addColumn(TableAddColumnState& addColumnState, PageAllocator& pageAllocator);
|
68
68
|
|
69
69
|
bool checkIfNodeHasRels(transaction::Transaction* transaction,
|
70
70
|
common::ValueVector* srcNodeIDVector) const;
|
@@ -146,9 +146,8 @@ private:
|
|
146
146
|
const VersionRecordHandler* getVersionRecordHandler(CSRNodeGroupScanSource source) const;
|
147
147
|
|
148
148
|
private:
|
149
|
-
|
150
|
-
|
151
|
-
MemoryManager* memoryManager;
|
149
|
+
Table& table;
|
150
|
+
MemoryManager* mm;
|
152
151
|
ShadowFile* shadowFile;
|
153
152
|
bool enableCompression;
|
154
153
|
PackedCSRInfo packedCSRInfo;
|
@@ -101,7 +101,6 @@ public:
|
|
101
101
|
common::transaction_t getStartTS() const { return startTS; }
|
102
102
|
common::transaction_t getCommitTS() const { return commitTS; }
|
103
103
|
int64_t getCurrentTS() const { return currentTS; }
|
104
|
-
main::ClientContext* getClientContext() const { return clientContext; }
|
105
104
|
|
106
105
|
void setForceCheckpoint() { forceCheckpoint = true; }
|
107
106
|
bool shouldAppendToUndoBuffer() const {
|
@@ -135,7 +135,7 @@ static void appendNewChunkedGroup(MemoryManager& mm, transaction::Transaction* t
|
|
135
135
|
relTable.pushInsertInfo(transaction, direction, nodeGroup, chunkedGroup.getNumRows(), source);
|
136
136
|
if (isNewNodeGroup) {
|
137
137
|
auto flushedChunkedGroup =
|
138
|
-
chunkedGroup.flushAsNewChunkedNodeGroup(transaction, pageAllocator);
|
138
|
+
chunkedGroup.flushAsNewChunkedNodeGroup(transaction, mm, pageAllocator);
|
139
139
|
|
140
140
|
// If there are deleted columns that haven't been vacuumed yet
|
141
141
|
// we need to add extra columns to the chunked group
|
@@ -26,7 +26,7 @@ std::vector<LogicalType> LocalNodeTable::getNodeTableColumnTypes(
|
|
26
26
|
LocalNodeTable::LocalNodeTable(const catalog::TableCatalogEntry* tableEntry, Table& table,
|
27
27
|
MemoryManager& mm)
|
28
28
|
: LocalTable{table}, overflowFileHandle(nullptr),
|
29
|
-
nodeGroups{getNodeTableColumnTypes(*tableEntry), false /*enableCompression*/} {
|
29
|
+
nodeGroups{mm, getNodeTableColumnTypes(*tableEntry), false /*enableCompression*/} {
|
30
30
|
initLocalHashIndex(mm);
|
31
31
|
startOffset = table.getNumTotalRows(nullptr /* transaction */);
|
32
32
|
}
|
@@ -70,7 +70,7 @@ bool LocalNodeTable::insert(Transaction* transaction, TableInsertState& insertSt
|
|
70
70
|
const auto nodeIDPos =
|
71
71
|
nodeInsertState.nodeIDVector.state->getSelVector().getSelectedPositions()[0];
|
72
72
|
nodeInsertState.nodeIDVector.setValue(nodeIDPos, internalID_t{nodeOffset, table.getTableID()});
|
73
|
-
nodeGroups.append(
|
73
|
+
nodeGroups.append(&DUMMY_TRANSACTION, insertState.propertyVectors);
|
74
74
|
return true;
|
75
75
|
}
|
76
76
|
|
@@ -104,8 +104,8 @@ bool LocalNodeTable::delete_(Transaction* transaction, TableDeleteState& deleteS
|
|
104
104
|
return nodeGroup->delete_(transaction, rowIdxInGroup);
|
105
105
|
}
|
106
106
|
|
107
|
-
bool LocalNodeTable::addColumn(
|
108
|
-
nodeGroups.addColumn(
|
107
|
+
bool LocalNodeTable::addColumn(TableAddColumnState& addColumnState) {
|
108
|
+
nodeGroups.addColumn(addColumnState);
|
109
109
|
return true;
|
110
110
|
}
|
111
111
|
|
@@ -25,17 +25,18 @@ static std::vector<LogicalType> getTypesForLocalRelTable(const catalog::TableCat
|
|
25
25
|
return types;
|
26
26
|
}
|
27
27
|
|
28
|
-
LocalRelTable::LocalRelTable(const catalog::TableCatalogEntry* tableEntry, const Table& table
|
28
|
+
LocalRelTable::LocalRelTable(const catalog::TableCatalogEntry* tableEntry, const Table& table,
|
29
|
+
MemoryManager& mm)
|
29
30
|
: LocalTable{table} {
|
30
|
-
localNodeGroup = std::make_unique<NodeGroup>(0, false,
|
31
|
-
INVALID_ROW_IDX);
|
31
|
+
localNodeGroup = std::make_unique<NodeGroup>(mm, 0, false,
|
32
|
+
getTypesForLocalRelTable(*tableEntry), INVALID_ROW_IDX);
|
32
33
|
const auto& relTable = table.cast<RelTable>();
|
33
34
|
for (auto relDirection : relTable.getStorageDirections()) {
|
34
35
|
directedIndices.emplace_back(relDirection);
|
35
36
|
}
|
36
37
|
}
|
37
38
|
|
38
|
-
bool LocalRelTable::insert(Transaction
|
39
|
+
bool LocalRelTable::insert(Transaction*, TableInsertState& state) {
|
39
40
|
const auto& insertState = state.cast<RelTableInsertState>();
|
40
41
|
|
41
42
|
std::vector<row_idx_vec_t*> rowIndicesToInsertTo;
|
@@ -64,7 +65,7 @@ bool LocalRelTable::insert(Transaction* transaction, TableInsertState& state) {
|
|
64
65
|
insertVectors.push_back(insertState.propertyVectors[i]);
|
65
66
|
}
|
66
67
|
const auto numRowsToAppend = insertState.srcNodeIDVector.state->getSelVector().getSelSize();
|
67
|
-
localNodeGroup->append(
|
68
|
+
localNodeGroup->append(&DUMMY_TRANSACTION, insertVectors, 0, numRowsToAppend);
|
68
69
|
|
69
70
|
for (auto* rowIndexToInsertTo : rowIndicesToInsertTo) {
|
70
71
|
rowIndexToInsertTo->push_back(numRowsInLocalTable);
|
@@ -137,14 +138,14 @@ bool LocalRelTable::delete_(Transaction* transaction, TableDeleteState& state) {
|
|
137
138
|
return true;
|
138
139
|
}
|
139
140
|
|
140
|
-
bool LocalRelTable::addColumn(
|
141
|
-
localNodeGroup->addColumn(
|
141
|
+
bool LocalRelTable::addColumn(TableAddColumnState& addColumnState) {
|
142
|
+
localNodeGroup->addColumn(addColumnState, nullptr /* FileHandle */,
|
142
143
|
nullptr /* newColumnStats */);
|
143
144
|
return true;
|
144
145
|
}
|
145
146
|
|
146
147
|
uint64_t LocalRelTable::getEstimatedMemUsage() {
|
147
|
-
//
|
148
|
+
// Estimation of fwdIndex and bwdIndex is rough.
|
148
149
|
if (!localNodeGroup) {
|
149
150
|
return 0;
|
150
151
|
}
|
@@ -18,18 +18,18 @@ LocalTable* LocalStorage::getOrCreateLocalTable(Table& table) {
|
|
18
18
|
const auto tableID = table.getTableID();
|
19
19
|
auto catalog = clientContext.getCatalog();
|
20
20
|
auto transaction = clientContext.getTransaction();
|
21
|
+
auto& mm = *clientContext.getMemoryManager();
|
21
22
|
if (!tables.contains(tableID)) {
|
22
23
|
switch (table.getTableType()) {
|
23
24
|
case TableType::NODE: {
|
24
25
|
auto tableEntry = catalog->getTableCatalogEntry(transaction, table.getTableID());
|
25
|
-
tables[tableID] = std::make_unique<LocalNodeTable>(tableEntry, table,
|
26
|
-
*clientContext.getMemoryManager());
|
26
|
+
tables[tableID] = std::make_unique<LocalNodeTable>(tableEntry, table, mm);
|
27
27
|
} break;
|
28
28
|
case TableType::REL: {
|
29
29
|
// We have to fetch the rel group entry from the catalog to based on the relGroupID.
|
30
30
|
auto tableEntry =
|
31
31
|
catalog->getTableCatalogEntry(transaction, table.cast<RelTable>().getRelGroupID());
|
32
|
-
tables[tableID] = std::make_unique<LocalRelTable>(tableEntry, table);
|
32
|
+
tables[tableID] = std::make_unique<LocalRelTable>(tableEntry, table, mm);
|
33
33
|
} break;
|
34
34
|
default:
|
35
35
|
KU_UNREACHABLE;
|
@@ -2,7 +2,6 @@
|
|
2
2
|
|
3
3
|
#include "common/assert.h"
|
4
4
|
#include "common/types/types.h"
|
5
|
-
#include "main/client_context.h"
|
6
5
|
#include "storage/buffer_manager/buffer_manager.h"
|
7
6
|
#include "storage/buffer_manager/memory_manager.h"
|
8
7
|
#include "storage/buffer_manager/spiller.h"
|
@@ -368,13 +367,11 @@ bool ChunkedNodeGroup::delete_(const Transaction* transaction, row_idx_t rowIdxI
|
|
368
367
|
return versionInfo->delete_(transaction->getID(), rowIdxInChunk);
|
369
368
|
}
|
370
369
|
|
371
|
-
void ChunkedNodeGroup::addColumn(const
|
372
|
-
|
373
|
-
ColumnStats* newColumnStats) {
|
370
|
+
void ChunkedNodeGroup::addColumn(MemoryManager& mm, const TableAddColumnState& addColumnState,
|
371
|
+
bool enableCompression, PageAllocator* pageAllocator, ColumnStats* newColumnStats) {
|
374
372
|
auto& dataType = addColumnState.propertyDefinition.getType();
|
375
|
-
chunks.push_back(
|
376
|
-
|
377
|
-
dataType.copy(), capacity, enableCompression, ResidencyState::IN_MEMORY));
|
373
|
+
chunks.push_back(std::make_unique<ColumnChunk>(mm, dataType.copy(), capacity, enableCompression,
|
374
|
+
ResidencyState::IN_MEMORY));
|
378
375
|
auto& chunkData = chunks.back()->getData();
|
379
376
|
auto numExistingRows = getNumRows();
|
380
377
|
chunkData.populateWithDefaultVal(addColumnState.defaultEvaluator, numExistingRows,
|
@@ -419,11 +416,12 @@ void ChunkedNodeGroup::finalize() const {
|
|
419
416
|
}
|
420
417
|
|
421
418
|
std::unique_ptr<ChunkedNodeGroup> ChunkedNodeGroup::flushAsNewChunkedNodeGroup(
|
422
|
-
Transaction* transaction, PageAllocator& pageAllocator) const {
|
419
|
+
Transaction* transaction, MemoryManager& mm, PageAllocator& pageAllocator) const {
|
423
420
|
std::vector<std::unique_ptr<ColumnChunk>> flushedChunks(getNumColumns());
|
424
421
|
for (auto i = 0u; i < getNumColumns(); i++) {
|
425
|
-
flushedChunks[i] =
|
426
|
-
|
422
|
+
flushedChunks[i] =
|
423
|
+
std::make_unique<ColumnChunk>(mm, getColumnChunk(i).isCompressionEnabled(),
|
424
|
+
Column::flushChunkData(getColumnChunk(i).getData(), pageAllocator));
|
427
425
|
}
|
428
426
|
auto flushedChunkedGroup =
|
429
427
|
std::make_unique<ChunkedNodeGroup>(std::move(flushedChunks), 0 /*startRowIdx*/);
|
@@ -5,7 +5,6 @@
|
|
5
5
|
#include "common/serializer/deserializer.h"
|
6
6
|
#include "common/serializer/serializer.h"
|
7
7
|
#include "common/vector/value_vector.h"
|
8
|
-
#include "main/client_context.h"
|
9
8
|
#include "storage/storage_utils.h"
|
10
9
|
#include "storage/table/column.h"
|
11
10
|
#include "transaction/transaction.h"
|
@@ -16,23 +15,24 @@ using namespace kuzu::transaction;
|
|
16
15
|
namespace kuzu {
|
17
16
|
namespace storage {
|
18
17
|
|
19
|
-
ColumnChunk::ColumnChunk(MemoryManager&
|
18
|
+
ColumnChunk::ColumnChunk(MemoryManager& mm, LogicalType&& dataType, uint64_t capacity,
|
20
19
|
bool enableCompression, ResidencyState residencyState, bool initializeToZero)
|
21
|
-
: enableCompression{enableCompression} {
|
22
|
-
data = ColumnChunkFactory::createColumnChunkData(
|
23
|
-
|
20
|
+
: mm{mm}, enableCompression{enableCompression} {
|
21
|
+
data = ColumnChunkFactory::createColumnChunkData(mm, std::move(dataType), enableCompression,
|
22
|
+
capacity, residencyState, true, initializeToZero);
|
24
23
|
KU_ASSERT(residencyState != ResidencyState::ON_DISK);
|
25
24
|
}
|
26
25
|
|
27
|
-
ColumnChunk::ColumnChunk(MemoryManager&
|
28
|
-
|
29
|
-
: enableCompression{enableCompression} {
|
30
|
-
data = ColumnChunkFactory::createColumnChunkData(
|
31
|
-
|
26
|
+
ColumnChunk::ColumnChunk(MemoryManager& mm, LogicalType&& dataType, bool enableCompression,
|
27
|
+
ColumnChunkMetadata metadata)
|
28
|
+
: mm{mm}, enableCompression{enableCompression} {
|
29
|
+
data = ColumnChunkFactory::createColumnChunkData(mm, std::move(dataType), enableCompression,
|
30
|
+
metadata, true, true);
|
32
31
|
}
|
33
32
|
|
34
|
-
ColumnChunk::ColumnChunk(bool enableCompression,
|
35
|
-
|
33
|
+
ColumnChunk::ColumnChunk(MemoryManager& mm, bool enableCompression,
|
34
|
+
std::unique_ptr<ColumnChunkData> data)
|
35
|
+
: mm{mm}, enableCompression{enableCompression}, data{std::move(data)} {}
|
36
36
|
|
37
37
|
void ColumnChunk::initializeScanState(ChunkState& state, const Column* column) const {
|
38
38
|
data->initializeScanState(state, column);
|
@@ -221,14 +221,13 @@ void ColumnChunk::serialize(Serializer& serializer) const {
|
|
221
221
|
data->serialize(serializer);
|
222
222
|
}
|
223
223
|
|
224
|
-
std::unique_ptr<ColumnChunk> ColumnChunk::deserialize(MemoryManager&
|
225
|
-
Deserializer& deSer) {
|
224
|
+
std::unique_ptr<ColumnChunk> ColumnChunk::deserialize(MemoryManager& mm, Deserializer& deSer) {
|
226
225
|
std::string key;
|
227
226
|
bool enableCompression = false;
|
228
227
|
deSer.validateDebuggingInfo(key, "enable_compression");
|
229
228
|
deSer.deserializeValue<bool>(enableCompression);
|
230
|
-
auto data = ColumnChunkData::deserialize(
|
231
|
-
return std::make_unique<ColumnChunk>(enableCompression, std::move(data));
|
229
|
+
auto data = ColumnChunkData::deserialize(mm, deSer);
|
230
|
+
return std::make_unique<ColumnChunk>(mm, enableCompression, std::move(data));
|
232
231
|
}
|
233
232
|
|
234
233
|
row_idx_t ColumnChunk::getNumUpdatedRows(const Transaction* transaction) const {
|
@@ -238,7 +237,6 @@ row_idx_t ColumnChunk::getNumUpdatedRows(const Transaction* transaction) const {
|
|
238
237
|
std::pair<std::unique_ptr<ColumnChunk>, std::unique_ptr<ColumnChunk>> ColumnChunk::scanUpdates(
|
239
238
|
const Transaction* transaction) const {
|
240
239
|
auto numUpdatedRows = getNumUpdatedRows(transaction);
|
241
|
-
auto& mm = *transaction->getClientContext()->getMemoryManager();
|
242
240
|
// TODO(Guodong): Actually for row idx in a column chunk, UINT32 should be enough.
|
243
241
|
auto updatedRows = std::make_unique<ColumnChunk>(mm, LogicalType::UINT64(), numUpdatedRows,
|
244
242
|
false, ResidencyState::IN_MEMORY);
|