tigerbeetle-node 0.11.8 → 0.11.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.client.node.sha256 +1 -1
- package/package.json +4 -3
- package/scripts/build_lib.sh +41 -0
- package/src/node.zig +1 -1
- package/src/tigerbeetle/scripts/validate_docs.sh +7 -1
- package/src/tigerbeetle/src/benchmark.zig +3 -3
- package/src/tigerbeetle/src/config.zig +31 -16
- package/src/tigerbeetle/src/constants.zig +48 -9
- package/src/tigerbeetle/src/ewah.zig +5 -5
- package/src/tigerbeetle/src/ewah_fuzz.zig +1 -1
- package/src/tigerbeetle/src/lsm/binary_search.zig +1 -1
- package/src/tigerbeetle/src/lsm/bloom_filter.zig +1 -1
- package/src/tigerbeetle/src/lsm/compaction.zig +34 -21
- package/src/tigerbeetle/src/lsm/forest_fuzz.zig +84 -104
- package/src/tigerbeetle/src/lsm/grid.zig +19 -13
- package/src/tigerbeetle/src/lsm/manifest_log.zig +8 -10
- package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig +18 -13
- package/src/tigerbeetle/src/lsm/merge_iterator.zig +1 -1
- package/src/tigerbeetle/src/lsm/segmented_array.zig +17 -17
- package/src/tigerbeetle/src/lsm/segmented_array_fuzz.zig +1 -1
- package/src/tigerbeetle/src/lsm/set_associative_cache.zig +1 -1
- package/src/tigerbeetle/src/lsm/table.zig +8 -20
- package/src/tigerbeetle/src/lsm/table_immutable.zig +1 -1
- package/src/tigerbeetle/src/lsm/table_iterator.zig +3 -3
- package/src/tigerbeetle/src/lsm/table_mutable.zig +14 -2
- package/src/tigerbeetle/src/lsm/test.zig +5 -4
- package/src/tigerbeetle/src/lsm/tree.zig +1 -2
- package/src/tigerbeetle/src/lsm/tree_fuzz.zig +85 -115
- package/src/tigerbeetle/src/message_bus.zig +4 -4
- package/src/tigerbeetle/src/message_pool.zig +7 -10
- package/src/tigerbeetle/src/ring_buffer.zig +22 -12
- package/src/tigerbeetle/src/simulator.zig +366 -239
- package/src/tigerbeetle/src/state_machine/auditor.zig +5 -5
- package/src/tigerbeetle/src/state_machine/workload.zig +3 -3
- package/src/tigerbeetle/src/state_machine.zig +190 -178
- package/src/tigerbeetle/src/{util.zig → stdx.zig} +2 -0
- package/src/tigerbeetle/src/storage.zig +13 -6
- package/src/tigerbeetle/src/{test → testing/cluster}/message_bus.zig +3 -3
- package/src/tigerbeetle/src/{test → testing/cluster}/network.zig +46 -22
- package/src/tigerbeetle/src/testing/cluster/state_checker.zig +169 -0
- package/src/tigerbeetle/src/testing/cluster/storage_checker.zig +202 -0
- package/src/tigerbeetle/src/testing/cluster.zig +443 -0
- package/src/tigerbeetle/src/{test → testing}/fuzz.zig +0 -0
- package/src/tigerbeetle/src/testing/hash_log.zig +66 -0
- package/src/tigerbeetle/src/{test → testing}/id.zig +0 -0
- package/src/tigerbeetle/src/testing/packet_simulator.zig +365 -0
- package/src/tigerbeetle/src/{test → testing}/priority_queue.zig +1 -1
- package/src/tigerbeetle/src/testing/reply_sequence.zig +139 -0
- package/src/tigerbeetle/src/{test → testing}/state_machine.zig +3 -1
- package/src/tigerbeetle/src/testing/storage.zig +757 -0
- package/src/tigerbeetle/src/{test → testing}/table.zig +21 -0
- package/src/tigerbeetle/src/{test → testing}/time.zig +0 -0
- package/src/tigerbeetle/src/tigerbeetle.zig +2 -0
- package/src/tigerbeetle/src/tracer.zig +3 -3
- package/src/tigerbeetle/src/unit_tests.zig +4 -4
- package/src/tigerbeetle/src/vopr.zig +2 -2
- package/src/tigerbeetle/src/vsr/client.zig +5 -2
- package/src/tigerbeetle/src/vsr/clock.zig +93 -53
- package/src/tigerbeetle/src/vsr/journal.zig +109 -98
- package/src/tigerbeetle/src/vsr/journal_format_fuzz.zig +2 -2
- package/src/tigerbeetle/src/vsr/replica.zig +1983 -1430
- package/src/tigerbeetle/src/vsr/replica_format.zig +13 -13
- package/src/tigerbeetle/src/vsr/superblock.zig +240 -142
- package/src/tigerbeetle/src/vsr/superblock_client_table.zig +7 -7
- package/src/tigerbeetle/src/vsr/superblock_free_set.zig +1 -1
- package/src/tigerbeetle/src/vsr/superblock_free_set_fuzz.zig +1 -1
- package/src/tigerbeetle/src/vsr/superblock_fuzz.zig +49 -14
- package/src/tigerbeetle/src/vsr/superblock_manifest.zig +38 -19
- package/src/tigerbeetle/src/vsr/superblock_quorums.zig +48 -48
- package/src/tigerbeetle/src/vsr/superblock_quorums_fuzz.zig +51 -51
- package/src/tigerbeetle/src/vsr.zig +99 -33
- package/src/tigerbeetle/src/demo.zig +0 -132
- package/src/tigerbeetle/src/demo_01_create_accounts.zig +0 -35
- package/src/tigerbeetle/src/demo_02_lookup_accounts.zig +0 -7
- package/src/tigerbeetle/src/demo_03_create_transfers.zig +0 -37
- package/src/tigerbeetle/src/demo_04_create_pending_transfers.zig +0 -61
- package/src/tigerbeetle/src/demo_05_post_pending_transfers.zig +0 -37
- package/src/tigerbeetle/src/demo_06_void_pending_transfers.zig +0 -24
- package/src/tigerbeetle/src/demo_07_lookup_transfers.zig +0 -7
- package/src/tigerbeetle/src/test/cluster.zig +0 -352
- package/src/tigerbeetle/src/test/conductor.zig +0 -366
- package/src/tigerbeetle/src/test/packet_simulator.zig +0 -398
- package/src/tigerbeetle/src/test/state_checker.zig +0 -169
- package/src/tigerbeetle/src/test/storage.zig +0 -864
- package/src/tigerbeetle/src/test/storage_checker.zig +0 -204
|
@@ -4,7 +4,7 @@ const allocator = testing.allocator;
|
|
|
4
4
|
const assert = std.debug.assert;
|
|
5
5
|
|
|
6
6
|
const constants = @import("../constants.zig");
|
|
7
|
-
const fuzz = @import("../
|
|
7
|
+
const fuzz = @import("../testing/fuzz.zig");
|
|
8
8
|
const vsr = @import("../vsr.zig");
|
|
9
9
|
|
|
10
10
|
const log = std.log.scoped(.lsm_forest_fuzz);
|
|
@@ -13,7 +13,7 @@ const tracer = @import("../tracer.zig");
|
|
|
13
13
|
const MessagePool = @import("../message_pool.zig").MessagePool;
|
|
14
14
|
const Transfer = @import("../tigerbeetle.zig").Transfer;
|
|
15
15
|
const Account = @import("../tigerbeetle.zig").Account;
|
|
16
|
-
const Storage = @import("../
|
|
16
|
+
const Storage = @import("../testing/storage.zig").Storage;
|
|
17
17
|
const StateMachine = @import("../state_machine.zig").StateMachineType(Storage, .{
|
|
18
18
|
.message_body_size_max = constants.message_body_size_max,
|
|
19
19
|
});
|
|
@@ -62,77 +62,48 @@ const Environment = struct {
|
|
|
62
62
|
) catch unreachable;
|
|
63
63
|
|
|
64
64
|
const State = enum {
|
|
65
|
-
uninit,
|
|
66
65
|
init,
|
|
67
|
-
|
|
66
|
+
superblock_format,
|
|
68
67
|
superblock_open,
|
|
68
|
+
forest_init,
|
|
69
69
|
forest_open,
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
fuzzing,
|
|
71
|
+
forest_compact,
|
|
72
|
+
forest_checkpoint,
|
|
73
|
+
superblock_checkpoint,
|
|
73
74
|
};
|
|
74
75
|
|
|
75
76
|
state: State,
|
|
76
77
|
storage: *Storage,
|
|
77
78
|
message_pool: MessagePool,
|
|
78
79
|
superblock: SuperBlock,
|
|
79
|
-
superblock_context: SuperBlock.Context
|
|
80
|
+
superblock_context: SuperBlock.Context,
|
|
80
81
|
grid: Grid,
|
|
81
82
|
forest: Forest,
|
|
82
|
-
|
|
83
|
-
forest_exists: bool,
|
|
84
|
-
checkpoint_op: ?u64 = null,
|
|
85
|
-
|
|
86
|
-
fn init(env: *Environment, storage: *Storage) !void {
|
|
87
|
-
env.state = .uninit;
|
|
83
|
+
checkpoint_op: ?u64,
|
|
88
84
|
|
|
85
|
+
pub fn run(storage: *Storage, fuzz_ops: []const FuzzOp) !void {
|
|
86
|
+
var env: Environment = undefined;
|
|
87
|
+
env.state = .init;
|
|
89
88
|
env.storage = storage;
|
|
90
|
-
errdefer env.storage.deinit(allocator);
|
|
91
89
|
|
|
92
90
|
env.message_pool = try MessagePool.init(allocator, .replica);
|
|
93
|
-
|
|
91
|
+
defer env.message_pool.deinit(allocator);
|
|
94
92
|
|
|
95
93
|
env.superblock = try SuperBlock.init(allocator, .{
|
|
96
94
|
.storage = env.storage,
|
|
97
95
|
.storage_size_limit = constants.storage_size_max,
|
|
98
96
|
.message_pool = &env.message_pool,
|
|
99
97
|
});
|
|
100
|
-
|
|
98
|
+
defer env.superblock.deinit(allocator);
|
|
101
99
|
|
|
102
100
|
env.grid = try Grid.init(allocator, &env.superblock);
|
|
103
|
-
|
|
101
|
+
defer env.grid.deinit(allocator);
|
|
104
102
|
|
|
105
|
-
// Forest must be initialized with an open superblock.
|
|
106
103
|
env.forest = undefined;
|
|
107
|
-
env.
|
|
108
|
-
|
|
109
|
-
env.state = .init;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
fn deinit(env: *Environment) void {
|
|
113
|
-
assert(env.state != .uninit);
|
|
114
|
-
|
|
115
|
-
if (env.forest_exists) {
|
|
116
|
-
env.forest.deinit(allocator);
|
|
117
|
-
env.forest_exists = false;
|
|
118
|
-
}
|
|
119
|
-
env.grid.deinit(allocator);
|
|
120
|
-
env.superblock.deinit(allocator);
|
|
121
|
-
env.message_pool.deinit(allocator);
|
|
122
|
-
|
|
123
|
-
env.state = .uninit;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
fn tick(env: *Environment) void {
|
|
127
|
-
// env.grid.tick();
|
|
128
|
-
env.storage.tick();
|
|
129
|
-
}
|
|
104
|
+
env.checkpoint_op = null;
|
|
130
105
|
|
|
131
|
-
|
|
132
|
-
// Sometimes IO completes synchronously (eg if cached), so we might already be in next_state before ticking.
|
|
133
|
-
assert(env.state == current_state or env.state == next_state);
|
|
134
|
-
while (env.state == current_state) env.tick();
|
|
135
|
-
assert(env.state == next_state);
|
|
106
|
+
try env.open_then_apply(fuzz_ops);
|
|
136
107
|
}
|
|
137
108
|
|
|
138
109
|
fn change_state(env: *Environment, current_state: State, next_state: State) void {
|
|
@@ -140,79 +111,86 @@ const Environment = struct {
|
|
|
140
111
|
env.state = next_state;
|
|
141
112
|
}
|
|
142
113
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
114
|
+
fn tick_until_state_change(env: *Environment, current_state: State, next_state: State) void {
|
|
115
|
+
// Sometimes operations complete synchronously so we might already be in next_state before ticking.
|
|
116
|
+
//assert(env.state == current_state or env.state == next_state);
|
|
117
|
+
while (env.state == current_state) env.storage.tick();
|
|
118
|
+
assert(env.state == next_state);
|
|
119
|
+
}
|
|
148
120
|
|
|
149
|
-
|
|
121
|
+
pub fn open_then_apply(env: *Environment, fuzz_ops: []const FuzzOp) !void {
|
|
122
|
+
env.change_state(.init, .superblock_format);
|
|
150
123
|
env.superblock.format(superblock_format_callback, &env.superblock_context, .{
|
|
151
124
|
.cluster = cluster,
|
|
152
125
|
.replica = replica,
|
|
153
126
|
});
|
|
154
|
-
|
|
127
|
+
|
|
128
|
+
env.tick_until_state_change(.superblock_format, .superblock_open);
|
|
129
|
+
env.superblock.open(superblock_open_callback, &env.superblock_context);
|
|
130
|
+
|
|
131
|
+
env.tick_until_state_change(.superblock_open, .forest_init);
|
|
132
|
+
env.forest = try Forest.init(allocator, &env.grid, node_count, forest_options);
|
|
133
|
+
defer env.forest.deinit(allocator);
|
|
134
|
+
|
|
135
|
+
env.change_state(.forest_init, .forest_open);
|
|
136
|
+
env.forest.open(forest_open_callback);
|
|
137
|
+
|
|
138
|
+
env.tick_until_state_change(.forest_open, .fuzzing);
|
|
139
|
+
try env.apply(fuzz_ops);
|
|
155
140
|
}
|
|
156
141
|
|
|
157
142
|
fn superblock_format_callback(superblock_context: *SuperBlock.Context) void {
|
|
158
143
|
const env = @fieldParentPtr(@This(), "superblock_context", superblock_context);
|
|
159
|
-
env.change_state(.
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
pub fn open(env: *Environment) void {
|
|
163
|
-
assert(env.state == .init);
|
|
164
|
-
env.superblock.open(superblock_open_callback, &env.superblock_context);
|
|
165
|
-
env.tick_until_state_change(.init, .forest_open);
|
|
144
|
+
env.change_state(.superblock_format, .superblock_open);
|
|
166
145
|
}
|
|
167
146
|
|
|
168
147
|
fn superblock_open_callback(superblock_context: *SuperBlock.Context) void {
|
|
169
148
|
const env = @fieldParentPtr(@This(), "superblock_context", superblock_context);
|
|
170
|
-
env.change_state(.
|
|
171
|
-
env.forest = Forest.init(allocator, &env.grid, node_count, forest_options) catch unreachable;
|
|
172
|
-
env.forest_exists = true;
|
|
173
|
-
env.forest.open(forest_open_callback);
|
|
149
|
+
env.change_state(.superblock_open, .forest_init);
|
|
174
150
|
}
|
|
175
151
|
|
|
176
152
|
fn forest_open_callback(forest: *Forest) void {
|
|
177
153
|
const env = @fieldParentPtr(@This(), "forest", forest);
|
|
178
|
-
env.change_state(.
|
|
154
|
+
env.change_state(.forest_open, .fuzzing);
|
|
179
155
|
}
|
|
180
156
|
|
|
181
157
|
pub fn compact(env: *Environment, op: u64) void {
|
|
182
|
-
env.change_state(.
|
|
158
|
+
env.change_state(.fuzzing, .forest_compact);
|
|
183
159
|
env.forest.compact(forest_compact_callback, op);
|
|
184
|
-
env.tick_until_state_change(.
|
|
160
|
+
env.tick_until_state_change(.forest_compact, .fuzzing);
|
|
185
161
|
}
|
|
186
162
|
|
|
187
163
|
fn forest_compact_callback(forest: *Forest) void {
|
|
188
164
|
const env = @fieldParentPtr(@This(), "forest", forest);
|
|
189
|
-
env.change_state(.
|
|
165
|
+
env.change_state(.forest_compact, .fuzzing);
|
|
190
166
|
}
|
|
191
167
|
|
|
192
168
|
pub fn checkpoint(env: *Environment, op: u64) void {
|
|
169
|
+
assert(env.checkpoint_op == null);
|
|
193
170
|
env.checkpoint_op = op - constants.lsm_batch_multiple;
|
|
194
|
-
|
|
171
|
+
|
|
172
|
+
env.change_state(.fuzzing, .forest_checkpoint);
|
|
195
173
|
env.forest.checkpoint(forest_checkpoint_callback);
|
|
196
|
-
env.tick_until_state_change(.
|
|
197
|
-
env.tick_until_state_change(.
|
|
174
|
+
env.tick_until_state_change(.forest_checkpoint, .superblock_checkpoint);
|
|
175
|
+
env.tick_until_state_change(.superblock_checkpoint, .fuzzing);
|
|
198
176
|
}
|
|
199
177
|
|
|
200
178
|
fn forest_checkpoint_callback(forest: *Forest) void {
|
|
201
179
|
const env = @fieldParentPtr(@This(), "forest", forest);
|
|
202
|
-
env.
|
|
180
|
+
const op = env.checkpoint_op.?;
|
|
181
|
+
env.checkpoint_op = null;
|
|
182
|
+
|
|
183
|
+
env.change_state(.forest_checkpoint, .superblock_checkpoint);
|
|
203
184
|
env.superblock.checkpoint(superblock_checkpoint_callback, &env.superblock_context, .{
|
|
204
185
|
.commit_min_checksum = env.superblock.working.vsr_state.commit_min_checksum + 1,
|
|
205
|
-
.commit_min =
|
|
206
|
-
.commit_max =
|
|
207
|
-
.view_normal = 0,
|
|
208
|
-
.view = 0,
|
|
186
|
+
.commit_min = op,
|
|
187
|
+
.commit_max = op + 1,
|
|
209
188
|
});
|
|
210
|
-
env.checkpoint_op = null;
|
|
211
189
|
}
|
|
212
190
|
|
|
213
191
|
fn superblock_checkpoint_callback(superblock_context: *SuperBlock.Context) void {
|
|
214
192
|
const env = @fieldParentPtr(@This(), "superblock_context", superblock_context);
|
|
215
|
-
env.change_state(.
|
|
193
|
+
env.change_state(.superblock_checkpoint, .fuzzing);
|
|
216
194
|
}
|
|
217
195
|
|
|
218
196
|
fn prefetch_account(env: *Environment, id: u128) void {
|
|
@@ -230,31 +208,27 @@ const Environment = struct {
|
|
|
230
208
|
groove.prefetch_setup(null);
|
|
231
209
|
groove.prefetch_enqueue(id);
|
|
232
210
|
groove.prefetch(Getter.prefetch_callback, &getter.prefetch_context);
|
|
233
|
-
while (!getter.finished) env.tick();
|
|
211
|
+
while (!getter.finished) env.storage.tick();
|
|
234
212
|
}
|
|
235
213
|
|
|
236
|
-
fn
|
|
237
|
-
var env: Environment = undefined;
|
|
238
|
-
|
|
239
|
-
try env.init(storage);
|
|
240
|
-
defer env.deinit();
|
|
241
|
-
|
|
242
|
-
// Open the superblock then forest.
|
|
243
|
-
env.open();
|
|
244
|
-
|
|
214
|
+
fn apply(env: *Environment, fuzz_ops: []const FuzzOp) !void {
|
|
245
215
|
// The forest should behave like a simple key-value data-structure.
|
|
246
216
|
// We'll compare it to a hash map.
|
|
247
217
|
var model = std.hash_map.AutoHashMap(u128, Account).init(allocator);
|
|
248
218
|
defer model.deinit();
|
|
249
219
|
|
|
250
220
|
for (fuzz_ops) |fuzz_op, fuzz_op_index| {
|
|
221
|
+
assert(env.state == .fuzzing);
|
|
251
222
|
log.debug("Running fuzz_ops[{}/{}] == {}", .{ fuzz_op_index, fuzz_ops.len, fuzz_op });
|
|
252
|
-
|
|
253
|
-
|
|
223
|
+
|
|
224
|
+
const storage_size_used = env.storage.size_used();
|
|
225
|
+
log.debug("storage.size_used = {}/{}", .{ storage_size_used, env.storage.size });
|
|
226
|
+
|
|
254
227
|
const model_size = model.count() * @sizeOf(Account);
|
|
255
228
|
log.debug("space_amplification = {d:.2}", .{
|
|
256
229
|
@intToFloat(f64, storage_size_used) / @intToFloat(f64, model_size),
|
|
257
230
|
});
|
|
231
|
+
|
|
258
232
|
// Apply fuzz_op to the forest and the model.
|
|
259
233
|
switch (fuzz_op) {
|
|
260
234
|
.compact => |compact| {
|
|
@@ -294,7 +268,6 @@ pub fn run_fuzz_ops(storage_options: Storage.Options, fuzz_ops: []const FuzzOp)
|
|
|
294
268
|
var storage = try Storage.init(allocator, constants.storage_size_max, storage_options);
|
|
295
269
|
defer storage.deinit(allocator);
|
|
296
270
|
|
|
297
|
-
try Environment.format(&storage);
|
|
298
271
|
try Environment.run(&storage, fuzz_ops);
|
|
299
272
|
}
|
|
300
273
|
|
|
@@ -328,8 +301,8 @@ pub fn generate_fuzz_ops(random: std.rand.Random, fuzz_op_count: usize) ![]const
|
|
|
328
301
|
log.info("puts_since_compact_max = {}", .{Environment.puts_since_compact_max});
|
|
329
302
|
log.info("compacts_per_checkpoint = {}", .{Environment.compacts_per_checkpoint});
|
|
330
303
|
|
|
331
|
-
var
|
|
332
|
-
defer
|
|
304
|
+
var id_to_account = std.hash_map.AutoHashMap(u128, Account).init(allocator);
|
|
305
|
+
defer id_to_account.deinit();
|
|
333
306
|
|
|
334
307
|
var op: u64 = 1;
|
|
335
308
|
var puts_since_compact: usize = 0;
|
|
@@ -359,12 +332,10 @@ pub fn generate_fuzz_ops(random: std.rand.Random, fuzz_op_count: usize) ![]const
|
|
|
359
332
|
},
|
|
360
333
|
.put_account => put_account: {
|
|
361
334
|
const id = random_id(random, u128);
|
|
362
|
-
|
|
363
|
-
const timestamp = id_to_timestamp.get(id) orelse fuzz_op_index;
|
|
364
|
-
try id_to_timestamp.put(id, timestamp);
|
|
365
|
-
break :put_account FuzzOp{ .put_account = Account{
|
|
335
|
+
var account = id_to_account.get(id) orelse Account{
|
|
366
336
|
.id = id,
|
|
367
|
-
|
|
337
|
+
// `timestamp` must be unique.
|
|
338
|
+
.timestamp = fuzz_op_index,
|
|
368
339
|
.user_data = random_id(random, u128),
|
|
369
340
|
.reserved = [_]u8{0} ** 48,
|
|
370
341
|
.ledger = random_id(random, u32),
|
|
@@ -373,11 +344,20 @@ pub fn generate_fuzz_ops(random: std.rand.Random, fuzz_op_count: usize) ![]const
|
|
|
373
344
|
.debits_must_not_exceed_credits = random.boolean(),
|
|
374
345
|
.credits_must_not_exceed_debits = random.boolean(),
|
|
375
346
|
},
|
|
376
|
-
.debits_pending =
|
|
377
|
-
.debits_posted =
|
|
378
|
-
.credits_pending =
|
|
379
|
-
.credits_posted =
|
|
380
|
-
}
|
|
347
|
+
.debits_pending = 0,
|
|
348
|
+
.debits_posted = 0,
|
|
349
|
+
.credits_pending = 0,
|
|
350
|
+
.credits_posted = 0,
|
|
351
|
+
};
|
|
352
|
+
|
|
353
|
+
// These are the only fields we are allowed to change on existing accounts.
|
|
354
|
+
account.debits_pending = random.int(u64);
|
|
355
|
+
account.debits_posted = random.int(u64);
|
|
356
|
+
account.credits_pending = random.int(u64);
|
|
357
|
+
account.credits_posted = random.int(u64);
|
|
358
|
+
|
|
359
|
+
try id_to_account.put(account.id, account);
|
|
360
|
+
break :put_account FuzzOp{ .put_account = account };
|
|
381
361
|
},
|
|
382
362
|
.get_account => FuzzOp{
|
|
383
363
|
.get_account = random_id(random, u128),
|
|
@@ -10,7 +10,7 @@ const SuperBlockType = vsr.SuperBlockType;
|
|
|
10
10
|
const FIFO = @import("../fifo.zig").FIFO;
|
|
11
11
|
const IOPS = @import("../iops.zig").IOPS;
|
|
12
12
|
const SetAssociativeCache = @import("set_associative_cache.zig").SetAssociativeCache;
|
|
13
|
-
const
|
|
13
|
+
const stdx = @import("../stdx.zig");
|
|
14
14
|
|
|
15
15
|
const log = std.log.scoped(.grid);
|
|
16
16
|
|
|
@@ -36,6 +36,15 @@ pub const BlockType = enum(u8) {
|
|
|
36
36
|
}
|
|
37
37
|
};
|
|
38
38
|
|
|
39
|
+
// Leave this outside GridType so we can call it from modules that don't know about Storage.
|
|
40
|
+
pub fn alloc_block(
|
|
41
|
+
allocator: mem.Allocator,
|
|
42
|
+
) !*align(constants.sector_size) [constants.block_size]u8 {
|
|
43
|
+
const block = try allocator.alignedAlloc(u8, constants.sector_size, constants.block_size);
|
|
44
|
+
mem.set(u8, block, 0);
|
|
45
|
+
return block[0..constants.block_size];
|
|
46
|
+
}
|
|
47
|
+
|
|
39
48
|
/// The Grid provides access to on-disk blocks (blobs of `block_size` bytes).
|
|
40
49
|
/// Each block is identified by an "address" (`u64`, beginning at 1).
|
|
41
50
|
///
|
|
@@ -180,11 +189,6 @@ pub fn GridType(comptime Storage: type) type {
|
|
|
180
189
|
};
|
|
181
190
|
}
|
|
182
191
|
|
|
183
|
-
pub fn alloc_block(allocator: mem.Allocator) !BlockPtr {
|
|
184
|
-
const block = try allocator.alignedAlloc(u8, constants.sector_size, block_size);
|
|
185
|
-
return block[0..block_size];
|
|
186
|
-
}
|
|
187
|
-
|
|
188
192
|
pub fn deinit(grid: *Grid, allocator: mem.Allocator) void {
|
|
189
193
|
for (&grid.read_iop_blocks) |block| allocator.free(block);
|
|
190
194
|
|
|
@@ -292,6 +296,12 @@ pub fn GridType(comptime Storage: type) type {
|
|
|
292
296
|
grid.assert_not_writing(address, block.*);
|
|
293
297
|
grid.assert_not_reading(address, block.*);
|
|
294
298
|
|
|
299
|
+
if (constants.verify) {
|
|
300
|
+
for (grid.cache_blocks) |cache_block| {
|
|
301
|
+
assert(cache_block != block.*);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
295
305
|
assert(grid.superblock.opened);
|
|
296
306
|
assert(!grid.superblock.free_set.is_free(address));
|
|
297
307
|
|
|
@@ -340,9 +350,7 @@ pub fn GridType(comptime Storage: type) type {
|
|
|
340
350
|
const cache_index = grid.cache.insert_index(&completed_write.address);
|
|
341
351
|
const cache_block = &grid.cache_blocks[cache_index];
|
|
342
352
|
std.mem.swap(BlockPtr, cache_block, completed_write.block);
|
|
343
|
-
|
|
344
|
-
std.mem.set(u8, completed_write.block.*, undefined);
|
|
345
|
-
}
|
|
353
|
+
std.mem.set(u8, completed_write.block.*, 0);
|
|
346
354
|
|
|
347
355
|
// Start a queued write if possible *before* calling the completed
|
|
348
356
|
// write's callback. This ensures that if the callback calls
|
|
@@ -462,9 +470,7 @@ pub fn GridType(comptime Storage: type) type {
|
|
|
462
470
|
const cache_index = grid.cache.insert_index(&read.address);
|
|
463
471
|
const cache_block = &grid.cache_blocks[cache_index];
|
|
464
472
|
std.mem.swap(BlockPtr, iop_block, cache_block);
|
|
465
|
-
|
|
466
|
-
std.mem.set(u8, iop_block.*, undefined);
|
|
467
|
-
}
|
|
473
|
+
std.mem.set(u8, iop_block.*, 0);
|
|
468
474
|
|
|
469
475
|
// Handoff the iop to a pending read or release it before resolving the callbacks below.
|
|
470
476
|
if (grid.read_pending_queue.pop()) |pending| {
|
|
@@ -556,7 +562,7 @@ pub fn GridType(comptime Storage: type) type {
|
|
|
556
562
|
}
|
|
557
563
|
|
|
558
564
|
fn verify_cached_read(grid: *Grid, address: u64, cached_block: BlockPtrConst) void {
|
|
559
|
-
if (Storage != @import("../
|
|
565
|
+
if (Storage != @import("../testing/storage.zig").Storage)
|
|
560
566
|
// Too complicated to do async verification
|
|
561
567
|
return;
|
|
562
568
|
|
|
@@ -28,11 +28,12 @@ const log = std.log.scoped(.manifest_log);
|
|
|
28
28
|
|
|
29
29
|
const constants = @import("../constants.zig");
|
|
30
30
|
const vsr = @import("../vsr.zig");
|
|
31
|
-
const
|
|
31
|
+
const stdx = @import("../stdx.zig");
|
|
32
32
|
|
|
33
33
|
const SuperBlockType = vsr.SuperBlockType;
|
|
34
34
|
const GridType = @import("grid.zig").GridType;
|
|
35
35
|
const BlockType = @import("grid.zig").BlockType;
|
|
36
|
+
const alloc_block = @import("grid.zig").alloc_block;
|
|
36
37
|
const tree = @import("tree.zig");
|
|
37
38
|
const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
|
|
38
39
|
|
|
@@ -93,7 +94,7 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
|
|
|
93
94
|
tree.compaction_tables_input_max + // Remove.
|
|
94
95
|
tree.compaction_tables_output_max);
|
|
95
96
|
|
|
96
|
-
const blocks_count_appends =
|
|
97
|
+
const blocks_count_appends = stdx.div_ceil(compaction_appends_max, Block.entry_count_max);
|
|
97
98
|
|
|
98
99
|
/// The upper-bound of manifest log blocks we must buffer.
|
|
99
100
|
///
|
|
@@ -151,10 +152,7 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
|
|
|
151
152
|
var blocks: [blocks_count_max]BlockPtr = undefined;
|
|
152
153
|
for (blocks) |*block, i| {
|
|
153
154
|
errdefer for (blocks[0..i]) |b| allocator.free(b);
|
|
154
|
-
|
|
155
|
-
const block_slice =
|
|
156
|
-
try allocator.alignedAlloc(u8, constants.sector_size, constants.block_size);
|
|
157
|
-
block.* = block_slice[0..constants.block_size];
|
|
155
|
+
block.* = try alloc_block(allocator);
|
|
158
156
|
}
|
|
159
157
|
errdefer for (blocks) |b| allocator.free(b);
|
|
160
158
|
|
|
@@ -255,7 +253,7 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
|
|
|
255
253
|
const label = labels_used[entry];
|
|
256
254
|
const table = &tables_used[entry];
|
|
257
255
|
|
|
258
|
-
if (manifest.insert_table_extent(table.address, block_reference.address, entry)) {
|
|
256
|
+
if (manifest.insert_table_extent(manifest_log.tree_hash, table.address, block_reference.address, entry)) {
|
|
259
257
|
switch (label.event) {
|
|
260
258
|
.insert => manifest_log.open_event(manifest_log, label.level, table),
|
|
261
259
|
.remove => manifest.queue_for_compaction(block_reference.address),
|
|
@@ -332,7 +330,7 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
|
|
|
332
330
|
|
|
333
331
|
const manifest: *SuperBlock.Manifest = &manifest_log.superblock.manifest;
|
|
334
332
|
const address = Block.address(block);
|
|
335
|
-
if (manifest.update_table_extent(table.address, address, entry)) |previous_block| {
|
|
333
|
+
if (manifest.update_table_extent(manifest_log.tree_hash, table.address, address, entry)) |previous_block| {
|
|
336
334
|
manifest.queue_for_compaction(previous_block);
|
|
337
335
|
if (label.event == .remove) manifest.queue_for_compaction(address);
|
|
338
336
|
} else {
|
|
@@ -433,6 +431,7 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
|
|
|
433
431
|
block,
|
|
434
432
|
address,
|
|
435
433
|
);
|
|
434
|
+
manifest_log.blocks.advance_head();
|
|
436
435
|
}
|
|
437
436
|
|
|
438
437
|
fn write_block_callback(write: *Grid.Write) void {
|
|
@@ -441,7 +440,6 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
|
|
|
441
440
|
assert(manifest_log.writing);
|
|
442
441
|
|
|
443
442
|
manifest_log.blocks_closed -= 1;
|
|
444
|
-
manifest_log.blocks.advance_head();
|
|
445
443
|
assert(manifest_log.blocks_closed <= manifest_log.blocks.count);
|
|
446
444
|
|
|
447
445
|
manifest_log.write_block();
|
|
@@ -547,7 +545,7 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
|
|
|
547
545
|
// Remove the extent if the table is the latest version.
|
|
548
546
|
// We must iterate entries in forward order to drop the extent here.
|
|
549
547
|
// Otherwise, stale versions earlier in the block may reappear.
|
|
550
|
-
if (manifest.remove_table_extent(table.address, block_reference.address, entry)) {
|
|
548
|
+
if (manifest.remove_table_extent(manifest_log.tree_hash, table.address, block_reference.address, entry)) {
|
|
551
549
|
switch (label.event) {
|
|
552
550
|
// Append the table, updating the table extent:
|
|
553
551
|
.insert => manifest_log.append(label, table),
|
|
@@ -22,11 +22,11 @@ const MessagePool = @import("../message_pool.zig").MessagePool;
|
|
|
22
22
|
const SuperBlock = @import("../vsr/superblock.zig").SuperBlockType(Storage);
|
|
23
23
|
const data_file_size_min = @import("../vsr/superblock.zig").data_file_size_min;
|
|
24
24
|
const TableExtent = @import("../vsr/superblock_manifest.zig").Manifest.TableExtent;
|
|
25
|
-
const Storage = @import("../
|
|
25
|
+
const Storage = @import("../testing/storage.zig").Storage;
|
|
26
26
|
const Grid = @import("grid.zig").GridType(Storage);
|
|
27
27
|
const BlockType = @import("grid.zig").BlockType;
|
|
28
28
|
const ManifestLog = @import("manifest_log.zig").ManifestLogType(Storage, TableInfo);
|
|
29
|
-
const fuzz = @import("../
|
|
29
|
+
const fuzz = @import("../testing/fuzz.zig");
|
|
30
30
|
|
|
31
31
|
pub const tigerbeetle_config = @import("../config.zig").configs.test_min;
|
|
32
32
|
|
|
@@ -198,7 +198,7 @@ fn generate_events(
|
|
|
198
198
|
event.* = switch (event_type) {
|
|
199
199
|
.insert_new => insert: {
|
|
200
200
|
const level = random.uintLessThan(u7, constants.lsm_levels);
|
|
201
|
-
const table =
|
|
201
|
+
const table = TableInfo{
|
|
202
202
|
.checksum = 0,
|
|
203
203
|
.address = i + 1,
|
|
204
204
|
.snapshot_min = 1,
|
|
@@ -210,10 +210,11 @@ fn generate_events(
|
|
|
210
210
|
.level = level,
|
|
211
211
|
.table = table,
|
|
212
212
|
});
|
|
213
|
-
|
|
213
|
+
const insert = ManifestEvent{ .insert = .{
|
|
214
214
|
.level = level,
|
|
215
215
|
.table = table,
|
|
216
216
|
} };
|
|
217
|
+
break :insert insert;
|
|
217
218
|
},
|
|
218
219
|
|
|
219
220
|
.insert_change_level => insert: {
|
|
@@ -223,27 +224,30 @@ fn generate_events(
|
|
|
223
224
|
}
|
|
224
225
|
|
|
225
226
|
table.level += 1;
|
|
226
|
-
|
|
227
|
+
const insert = ManifestEvent{ .insert = .{
|
|
227
228
|
.level = table.level,
|
|
228
229
|
.table = table.table,
|
|
229
230
|
} };
|
|
231
|
+
break :insert insert;
|
|
230
232
|
},
|
|
231
233
|
|
|
232
234
|
.insert_change_snapshot => insert: {
|
|
233
235
|
const table = &tables.items[random.uintLessThan(usize, tables.items.len)];
|
|
234
236
|
table.table.snapshot_max += 1;
|
|
235
|
-
|
|
237
|
+
const insert = ManifestEvent{ .insert = .{
|
|
236
238
|
.level = table.level,
|
|
237
239
|
.table = table.table,
|
|
238
240
|
} };
|
|
241
|
+
break :insert insert;
|
|
239
242
|
},
|
|
240
243
|
|
|
241
244
|
.remove => remove: {
|
|
242
245
|
const table = tables.swapRemove(random.uintLessThan(usize, tables.items.len));
|
|
243
|
-
|
|
246
|
+
const remove = ManifestEvent{ .remove = .{
|
|
244
247
|
.level = table.level,
|
|
245
248
|
.table = table.table,
|
|
246
249
|
} };
|
|
250
|
+
break :remove remove;
|
|
247
251
|
},
|
|
248
252
|
|
|
249
253
|
.compact => ManifestEvent{ .compact = {} },
|
|
@@ -415,16 +419,17 @@ const Environment = struct {
|
|
|
415
419
|
env.manifest_log.checkpoint(checkpoint_callback);
|
|
416
420
|
env.wait(&env.manifest_log);
|
|
417
421
|
|
|
418
|
-
|
|
419
|
-
vsr_state.commit_min += 1;
|
|
420
|
-
vsr_state.commit_min_checksum += 1;
|
|
421
|
-
vsr_state.commit_max += 1;
|
|
422
|
+
const vsr_state = &env.manifest_log.superblock.working.vsr_state;
|
|
422
423
|
|
|
423
424
|
env.pending += 1;
|
|
424
425
|
env.manifest_log.superblock.checkpoint(
|
|
425
426
|
checkpoint_superblock_callback,
|
|
426
427
|
&env.superblock_context,
|
|
427
|
-
|
|
428
|
+
.{
|
|
429
|
+
.commit_min_checksum = vsr_state.commit_min_checksum + 1,
|
|
430
|
+
.commit_min = vsr_state.commit_min + 1,
|
|
431
|
+
.commit_max = vsr_state.commit_max + 1,
|
|
432
|
+
},
|
|
428
433
|
);
|
|
429
434
|
env.wait(&env.manifest_log);
|
|
430
435
|
|
|
@@ -615,7 +620,7 @@ fn verify_manifest(
|
|
|
615
620
|
try std.testing.expect(std.mem.eql(u64, expect.addresses[0..c], actual.addresses[0..c]));
|
|
616
621
|
|
|
617
622
|
try std.testing.expect(hash_map_equals(
|
|
618
|
-
|
|
623
|
+
SuperBlock.Manifest.TableExtentKey,
|
|
619
624
|
SuperBlock.Manifest.TableExtent,
|
|
620
625
|
&expect.tables,
|
|
621
626
|
&actual.tables,
|