tigerbeetle-node 0.11.4 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.client.node.sha256 +1 -1
- package/dist/index.d.ts +41 -42
- package/dist/index.js +41 -42
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/index.ts +0 -1
- package/src/node.zig +5 -5
- package/src/tigerbeetle/scripts/benchmark.bat +6 -1
- package/src/tigerbeetle/scripts/benchmark.sh +1 -1
- package/src/tigerbeetle/src/benchmark.zig +4 -4
- package/src/tigerbeetle/src/c/tb_client/context.zig +6 -6
- package/src/tigerbeetle/src/c/tb_client/echo_client.zig +2 -2
- package/src/tigerbeetle/src/c/tb_client/thread.zig +0 -1
- package/src/tigerbeetle/src/c/tb_client.h +42 -43
- package/src/tigerbeetle/src/c/tb_client.zig +2 -2
- package/src/tigerbeetle/src/c/test.zig +8 -8
- package/src/tigerbeetle/src/cli.zig +41 -17
- package/src/tigerbeetle/src/config.zig +24 -3
- package/src/tigerbeetle/src/constants.zig +8 -5
- package/src/tigerbeetle/src/demo.zig +4 -4
- package/src/tigerbeetle/src/io/darwin.zig +4 -4
- package/src/tigerbeetle/src/io/linux.zig +6 -6
- package/src/tigerbeetle/src/io/windows.zig +4 -4
- package/src/tigerbeetle/src/lsm/compaction.zig +17 -51
- package/src/tigerbeetle/src/lsm/forest.zig +2 -2
- package/src/tigerbeetle/src/lsm/forest_fuzz.zig +11 -16
- package/src/tigerbeetle/src/lsm/grid.zig +5 -5
- package/src/tigerbeetle/src/lsm/groove.zig +7 -4
- package/src/tigerbeetle/src/lsm/level_iterator.zig +2 -2
- package/src/tigerbeetle/src/lsm/manifest.zig +19 -18
- package/src/tigerbeetle/src/lsm/manifest_level.zig +3 -2
- package/src/tigerbeetle/src/lsm/manifest_log.zig +8 -8
- package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig +31 -15
- package/src/tigerbeetle/src/lsm/merge_iterator.zig +106 -0
- package/src/tigerbeetle/src/lsm/posted_groove.zig +4 -3
- package/src/tigerbeetle/src/lsm/segmented_array.zig +1 -0
- package/src/tigerbeetle/src/lsm/segmented_array_benchmark.zig +13 -13
- package/src/tigerbeetle/src/lsm/set_associative_cache.zig +2 -2
- package/src/tigerbeetle/src/lsm/table.zig +33 -19
- package/src/tigerbeetle/src/lsm/table_immutable.zig +4 -4
- package/src/tigerbeetle/src/lsm/table_iterator.zig +19 -11
- package/src/tigerbeetle/src/lsm/table_mutable.zig +52 -18
- package/src/tigerbeetle/src/lsm/test.zig +14 -10
- package/src/tigerbeetle/src/lsm/tree.zig +48 -45
- package/src/tigerbeetle/src/lsm/tree_fuzz.zig +310 -271
- package/src/tigerbeetle/src/main.zig +40 -36
- package/src/tigerbeetle/src/message_bus.zig +25 -25
- package/src/tigerbeetle/src/message_pool.zig +18 -17
- package/src/tigerbeetle/src/simulator.zig +28 -83
- package/src/tigerbeetle/src/{test/accounting → state_machine}/auditor.zig +9 -9
- package/src/tigerbeetle/src/{test/accounting → state_machine}/workload.zig +112 -52
- package/src/tigerbeetle/src/state_machine.zig +20 -14
- package/src/tigerbeetle/src/storage.zig +12 -12
- package/src/tigerbeetle/src/test/cluster.zig +17 -17
- package/src/tigerbeetle/src/test/conductor.zig +8 -8
- package/src/tigerbeetle/src/test/id.zig +10 -0
- package/src/tigerbeetle/src/test/message_bus.zig +0 -2
- package/src/tigerbeetle/src/test/network.zig +5 -5
- package/src/tigerbeetle/src/test/state_checker.zig +2 -2
- package/src/tigerbeetle/src/test/state_machine.zig +151 -46
- package/src/tigerbeetle/src/test/storage.zig +54 -51
- package/src/tigerbeetle/src/test/storage_checker.zig +3 -3
- package/src/tigerbeetle/src/tigerbeetle.zig +0 -1
- package/src/tigerbeetle/src/time.zig +0 -1
- package/src/tigerbeetle/src/tracer.zig +4 -4
- package/src/tigerbeetle/src/unit_tests.zig +2 -2
- package/src/tigerbeetle/src/vsr/client.zig +10 -10
- package/src/tigerbeetle/src/vsr/clock.zig +11 -10
- package/src/tigerbeetle/src/vsr/journal.zig +581 -531
- package/src/tigerbeetle/src/vsr/journal_format_fuzz.zig +11 -11
- package/src/tigerbeetle/src/vsr/replica.zig +377 -367
- package/src/tigerbeetle/src/vsr/replica_format.zig +14 -11
- package/src/tigerbeetle/src/vsr/superblock.zig +125 -80
- package/src/tigerbeetle/src/vsr/superblock_client_table.zig +9 -9
- package/src/tigerbeetle/src/vsr/superblock_free_set.zig +4 -4
- package/src/tigerbeetle/src/vsr/superblock_fuzz.zig +11 -6
- package/src/tigerbeetle/src/vsr/superblock_manifest.zig +5 -5
- package/src/tigerbeetle/src/vsr/superblock_quorums_fuzz.zig +3 -3
- package/src/tigerbeetle/src/vsr.zig +25 -25
|
@@ -6,7 +6,7 @@ const testing = std.testing;
|
|
|
6
6
|
const c = @cImport(@cInclude("tb_client.h"));
|
|
7
7
|
|
|
8
8
|
const util = @import("../util.zig");
|
|
9
|
-
const
|
|
9
|
+
const constants = @import("../constants.zig");
|
|
10
10
|
const Packet = @import("tb_client/packet.zig").Packet;
|
|
11
11
|
|
|
12
12
|
const Mutex = std.Thread.Mutex;
|
|
@@ -91,10 +91,10 @@ const Completion = struct {
|
|
|
91
91
|
// 3. the data marshaling is correct, and exactly the same data sent was received back.
|
|
92
92
|
test "c_client echo" {
|
|
93
93
|
// Using the create_accounts operation for this test.
|
|
94
|
-
const RequestContext = RequestContextType(
|
|
94
|
+
const RequestContext = RequestContextType(constants.message_body_size_max);
|
|
95
95
|
const create_accounts_operation: u8 = c.TB_OPERATION_CREATE_ACCOUNTS;
|
|
96
96
|
const event_size = @sizeOf(c.tb_account_t);
|
|
97
|
-
const event_request_max = @divFloor(
|
|
97
|
+
const event_request_max = @divFloor(constants.message_body_size_max, event_size);
|
|
98
98
|
|
|
99
99
|
// Initializing an echo client for testing purposes.
|
|
100
100
|
// We ensure that the retry mechanism is being tested
|
|
@@ -103,7 +103,7 @@ test "c_client echo" {
|
|
|
103
103
|
var tb_packet_list: c.tb_packet_list_t = undefined;
|
|
104
104
|
const cluster_id = 0;
|
|
105
105
|
const address = "3000";
|
|
106
|
-
const packets_count: u32 =
|
|
106
|
+
const packets_count: u32 = constants.client_request_queue_max * 2;
|
|
107
107
|
const tb_context: usize = 42;
|
|
108
108
|
const result = c.tb_client_init_echo(
|
|
109
109
|
&tb_client,
|
|
@@ -221,7 +221,7 @@ test "c_client tb_status" {
|
|
|
221
221
|
// More addresses thant "replicas_max" should return "TB_STATUS_ADDRESS_LIMIT_EXCEEDED":
|
|
222
222
|
try assert_status(
|
|
223
223
|
1,
|
|
224
|
-
("3000," **
|
|
224
|
+
("3000," ** constants.replicas_max) ++ "3001",
|
|
225
225
|
c.TB_STATUS_ADDRESS_LIMIT_EXCEEDED,
|
|
226
226
|
);
|
|
227
227
|
|
|
@@ -234,7 +234,7 @@ test "c_client tb_status" {
|
|
|
234
234
|
|
|
235
235
|
// Asserts the validation rules associated with the "TB_PACKET_STATUS" enum.
|
|
236
236
|
test "c_client tb_packet_status" {
|
|
237
|
-
const RequestContext = RequestContextType(
|
|
237
|
+
const RequestContext = RequestContextType(constants.message_body_size_max);
|
|
238
238
|
|
|
239
239
|
var tb_client: c.tb_client_t = undefined;
|
|
240
240
|
var tb_packet_list: c.tb_packet_list_t = undefined;
|
|
@@ -299,12 +299,12 @@ test "c_client tb_packet_status" {
|
|
|
299
299
|
|
|
300
300
|
var packet_list = @ptrCast(*Packet.List, &tb_packet_list);
|
|
301
301
|
|
|
302
|
-
// Messages larger than
|
|
302
|
+
// Messages larger than constants.message_body_size_max should return "too_much_data":
|
|
303
303
|
try assert_result(
|
|
304
304
|
tb_client,
|
|
305
305
|
packet_list,
|
|
306
306
|
c.TB_OPERATION_CREATE_TRANSFERS,
|
|
307
|
-
|
|
307
|
+
constants.message_body_size_max + @sizeOf(c.tb_transfer_t),
|
|
308
308
|
c.TB_PACKET_TOO_MUCH_DATA,
|
|
309
309
|
);
|
|
310
310
|
|
|
@@ -7,12 +7,13 @@ const meta = std.meta;
|
|
|
7
7
|
const net = std.net;
|
|
8
8
|
const os = std.os;
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
const constants = @import("constants.zig");
|
|
11
11
|
const tigerbeetle = @import("tigerbeetle.zig");
|
|
12
12
|
const vsr = @import("vsr.zig");
|
|
13
13
|
const IO = @import("io.zig").IO;
|
|
14
|
+
const data_file_size_min = @import("vsr/superblock.zig").data_file_size_min;
|
|
14
15
|
|
|
15
|
-
// TODO Document --cache-accounts, --cache-transfers, --cache-transfers-posted
|
|
16
|
+
// TODO Document --cache-accounts, --cache-transfers, --cache-transfers-posted, --limit-storage
|
|
16
17
|
const usage = fmt.comptimePrint(
|
|
17
18
|
\\Usage:
|
|
18
19
|
\\
|
|
@@ -72,25 +73,28 @@ const usage = fmt.comptimePrint(
|
|
|
72
73
|
\\ tigerbeetle version --verbose
|
|
73
74
|
\\
|
|
74
75
|
, .{
|
|
75
|
-
.default_address =
|
|
76
|
-
.default_port =
|
|
76
|
+
.default_address = constants.address,
|
|
77
|
+
.default_port = constants.port,
|
|
77
78
|
});
|
|
78
79
|
|
|
79
80
|
pub const Command = union(enum) {
|
|
80
|
-
|
|
81
|
-
args_allocated: std.ArrayList([:0]const u8),
|
|
82
|
-
cluster: u32,
|
|
83
|
-
replica: u8,
|
|
84
|
-
path: [:0]const u8,
|
|
85
|
-
},
|
|
86
|
-
start: struct {
|
|
81
|
+
pub const Start = struct {
|
|
87
82
|
args_allocated: std.ArrayList([:0]const u8),
|
|
88
83
|
addresses: []net.Address,
|
|
89
84
|
cache_accounts: u32,
|
|
90
85
|
cache_transfers: u32,
|
|
91
86
|
cache_transfers_posted: u32,
|
|
87
|
+
storage_size_limit: u64,
|
|
88
|
+
path: [:0]const u8,
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
format: struct {
|
|
92
|
+
args_allocated: std.ArrayList([:0]const u8),
|
|
93
|
+
cluster: u32,
|
|
94
|
+
replica: u8,
|
|
92
95
|
path: [:0]const u8,
|
|
93
96
|
},
|
|
97
|
+
start: Start,
|
|
94
98
|
version: struct {
|
|
95
99
|
verbose: bool,
|
|
96
100
|
},
|
|
@@ -117,6 +121,7 @@ pub fn parse_args(allocator: std.mem.Allocator) !Command {
|
|
|
117
121
|
var cache_accounts: ?[]const u8 = null;
|
|
118
122
|
var cache_transfers: ?[]const u8 = null;
|
|
119
123
|
var cache_transfers_posted: ?[]const u8 = null;
|
|
124
|
+
var storage_size_limit: ?[]const u8 = null;
|
|
120
125
|
var verbose: ?bool = null;
|
|
121
126
|
|
|
122
127
|
var args = try std.process.argsWithAllocator(allocator);
|
|
@@ -163,6 +168,9 @@ pub fn parse_args(allocator: std.mem.Allocator) !Command {
|
|
|
163
168
|
} else if (mem.startsWith(u8, arg, "--cache-transfers-posted")) {
|
|
164
169
|
if (command != .start) fatal("--cache-transfers-posted: supported only by 'start' command", .{});
|
|
165
170
|
cache_transfers_posted = parse_flag("--cache-transfers-posted", arg);
|
|
171
|
+
} else if (mem.startsWith(u8, arg, "--limit-storage")) {
|
|
172
|
+
if (command != .start) fatal("--limit-storage: supported only by 'start' command", .{});
|
|
173
|
+
storage_size_limit = parse_flag("--limit-storage", arg);
|
|
166
174
|
} else if (mem.eql(u8, arg, "--verbose")) {
|
|
167
175
|
if (command != .version) fatal("--verbose: supported only by 'version' command", .{});
|
|
168
176
|
verbose = true;
|
|
@@ -206,18 +214,19 @@ pub fn parse_args(allocator: std.mem.Allocator) !Command {
|
|
|
206
214
|
.cache_accounts = parse_size_to_count(
|
|
207
215
|
tigerbeetle.Account,
|
|
208
216
|
cache_accounts,
|
|
209
|
-
|
|
217
|
+
constants.cache_accounts_max,
|
|
210
218
|
),
|
|
211
219
|
.cache_transfers = parse_size_to_count(
|
|
212
220
|
tigerbeetle.Transfer,
|
|
213
221
|
cache_transfers,
|
|
214
|
-
|
|
222
|
+
constants.cache_transfers_max,
|
|
215
223
|
),
|
|
216
224
|
.cache_transfers_posted = parse_size_to_count(
|
|
217
225
|
u256, // TODO(#264): Use actual type here, once exposed.
|
|
218
226
|
cache_transfers_posted,
|
|
219
|
-
|
|
227
|
+
constants.cache_transfers_posted_max,
|
|
220
228
|
),
|
|
229
|
+
.storage_size_limit = parse_storage_size(storage_size_limit),
|
|
221
230
|
.path = path orelse fatal("required: <path>", .{}),
|
|
222
231
|
},
|
|
223
232
|
};
|
|
@@ -255,11 +264,11 @@ fn parse_cluster(raw_cluster: []const u8) u32 {
|
|
|
255
264
|
|
|
256
265
|
/// Parse and allocate the addresses returning a slice into that array.
|
|
257
266
|
fn parse_addresses(allocator: std.mem.Allocator, raw_addresses: []const u8) []net.Address {
|
|
258
|
-
return vsr.parse_addresses(allocator, raw_addresses,
|
|
267
|
+
return vsr.parse_addresses(allocator, raw_addresses, constants.replicas_max) catch |err| switch (err) {
|
|
259
268
|
error.AddressHasTrailingComma => fatal("--addresses: invalid trailing comma", .{}),
|
|
260
269
|
error.AddressLimitExceeded => {
|
|
261
270
|
fatal("--addresses: too many addresses, at most {d} are allowed", .{
|
|
262
|
-
|
|
271
|
+
constants.replicas_max,
|
|
263
272
|
});
|
|
264
273
|
},
|
|
265
274
|
error.AddressHasMoreThanOneColon => {
|
|
@@ -272,6 +281,21 @@ fn parse_addresses(allocator: std.mem.Allocator, raw_addresses: []const u8) []ne
|
|
|
272
281
|
};
|
|
273
282
|
}
|
|
274
283
|
|
|
284
|
+
fn parse_storage_size(size_string: ?[]const u8) u64 {
|
|
285
|
+
const size_min = data_file_size_min;
|
|
286
|
+
const size_max = constants.storage_size_max;
|
|
287
|
+
const size = if (size_string) |s| parse_size(s) else size_max;
|
|
288
|
+
if (size > size_max) fatal("storage size {} exceeds maximum: {}", .{ size, size_max });
|
|
289
|
+
if (size < size_min) fatal("storage size {} is below minimum: {}", .{ size, size_min });
|
|
290
|
+
if (size % constants.sector_size != 0) {
|
|
291
|
+
fatal("size value {} must be a multiple of sector size ({})", .{
|
|
292
|
+
size,
|
|
293
|
+
constants.sector_size,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
return size;
|
|
297
|
+
}
|
|
298
|
+
|
|
275
299
|
fn parse_size(string: []const u8) u64 {
|
|
276
300
|
var value = mem.trim(u8, string, " ");
|
|
277
301
|
|
|
@@ -366,7 +390,7 @@ fn parse_size_to_count(comptime T: type, string_opt: ?[]const u8, comptime defau
|
|
|
366
390
|
}
|
|
367
391
|
|
|
368
392
|
fn parse_replica(raw_replica: []const u8) u8 {
|
|
369
|
-
comptime assert(
|
|
393
|
+
comptime assert(constants.replicas_max <= std.math.maxInt(u8));
|
|
370
394
|
const replica = fmt.parseUnsigned(u8, raw_replica, 10) catch |err| switch (err) {
|
|
371
395
|
error.Overflow => fatal("--replica: value exceeds an 8-bit unsigned integer", .{}),
|
|
372
396
|
error.InvalidCharacter => fatal("--replica: value contains an invalid character", .{}),
|
|
@@ -21,7 +21,7 @@ pub const Config = struct {
|
|
|
21
21
|
/// - Client configs need not equal replica configs.
|
|
22
22
|
/// - Replica configs can change between restarts.
|
|
23
23
|
///
|
|
24
|
-
/// Fields are documented within
|
|
24
|
+
/// Fields are documented within constants.zig.
|
|
25
25
|
const ConfigProcess = struct {
|
|
26
26
|
log_level: std.log.Level = .info,
|
|
27
27
|
tracer_backend: TracerBackend = .none,
|
|
@@ -64,7 +64,7 @@ const ConfigProcess = struct {
|
|
|
64
64
|
/// over the cluster lifetime.
|
|
65
65
|
/// - The storage formats generated by different ConfigClusters are incompatible.
|
|
66
66
|
///
|
|
67
|
-
/// Fields are documented within
|
|
67
|
+
/// Fields are documented within constants.zig.
|
|
68
68
|
const ConfigCluster = struct {
|
|
69
69
|
cache_line_size: comptime_int = 64,
|
|
70
70
|
clients_max: usize,
|
|
@@ -72,7 +72,7 @@ const ConfigCluster = struct {
|
|
|
72
72
|
journal_slot_count: usize = 1024,
|
|
73
73
|
message_size_max: usize = 1 * 1024 * 1024,
|
|
74
74
|
superblock_copies: comptime_int = 4,
|
|
75
|
-
|
|
75
|
+
storage_size_max: u64 = 16 * 1024 * 1024 * 1024 * 1024,
|
|
76
76
|
block_size: comptime_int = 64 * 1024,
|
|
77
77
|
lsm_levels: u7 = 7,
|
|
78
78
|
lsm_growth_factor: u32 = 8,
|
|
@@ -80,6 +80,7 @@ const ConfigCluster = struct {
|
|
|
80
80
|
lsm_batch_multiple: comptime_int = 4,
|
|
81
81
|
lsm_snapshots_max: usize = 32,
|
|
82
82
|
lsm_value_to_key_layout_ratio_min: comptime_int = 16,
|
|
83
|
+
state_machine: StateMachine = .accounting,
|
|
83
84
|
|
|
84
85
|
/// The WAL requires at least two sectors of redundant headers — otherwise we could lose them all to
|
|
85
86
|
/// a single torn write. A replica needs at least one valid redundant header to determine an
|
|
@@ -102,6 +103,13 @@ const ConfigCluster = struct {
|
|
|
102
103
|
}
|
|
103
104
|
};
|
|
104
105
|
|
|
106
|
+
pub const ConfigBase = enum {
|
|
107
|
+
production,
|
|
108
|
+
development,
|
|
109
|
+
test_min,
|
|
110
|
+
default,
|
|
111
|
+
};
|
|
112
|
+
|
|
105
113
|
pub const TracerBackend = enum {
|
|
106
114
|
none,
|
|
107
115
|
// Writes to a file (./tracer.json) which can be uploaded to https://ui.perfetto.dev/
|
|
@@ -110,6 +118,11 @@ pub const TracerBackend = enum {
|
|
|
110
118
|
tracy,
|
|
111
119
|
};
|
|
112
120
|
|
|
121
|
+
pub const StateMachine = enum {
|
|
122
|
+
accounting,
|
|
123
|
+
testing,
|
|
124
|
+
};
|
|
125
|
+
|
|
113
126
|
pub const configs = struct {
|
|
114
127
|
/// A good default config for production.
|
|
115
128
|
pub const default_production = Config{
|
|
@@ -156,6 +169,7 @@ pub const configs = struct {
|
|
|
156
169
|
.clients_max = 4,
|
|
157
170
|
.journal_slot_count = Config.Cluster.journal_slot_count_min,
|
|
158
171
|
.message_size_max = Config.Cluster.message_size_max_min(2),
|
|
172
|
+
.storage_size_max = 1024 * 1024 * 1024,
|
|
159
173
|
|
|
160
174
|
.block_size = sector_size,
|
|
161
175
|
.lsm_growth_factor = 4,
|
|
@@ -182,6 +196,13 @@ pub const configs = struct {
|
|
|
182
196
|
.test_min => test_min,
|
|
183
197
|
};
|
|
184
198
|
|
|
199
|
+
base.cluster.state_machine = if (@hasDecl(root, "decode_events"))
|
|
200
|
+
// TODO(DJ) This is a hack to work around the absense of tigerbeetle_build_options.
|
|
201
|
+
// This should be removed once the node client is built using `zig build`.
|
|
202
|
+
.accounting
|
|
203
|
+
else
|
|
204
|
+
@intToEnum(StateMachine, @enumToInt(build_options.config_cluster_state_machine));
|
|
205
|
+
|
|
185
206
|
// TODO Use additional build options to overwrite other fields.
|
|
186
207
|
base.process.tracer_backend = if (@hasDecl(root, "tracer_backend"))
|
|
187
208
|
// TODO(jamii)
|
|
@@ -21,6 +21,12 @@ pub const tracer_backend = config.process.tracer_backend;
|
|
|
21
21
|
/// The maximum number of replicas allowed in a cluster.
|
|
22
22
|
pub const replicas_max = 6;
|
|
23
23
|
|
|
24
|
+
pub const state_machine = config.cluster.state_machine;
|
|
25
|
+
pub const StateMachineType = switch (config.cluster.state_machine) {
|
|
26
|
+
.accounting => @import("state_machine.zig").StateMachineType,
|
|
27
|
+
.testing => @import("test/state_machine.zig").StateMachineType,
|
|
28
|
+
};
|
|
29
|
+
|
|
24
30
|
/// The maximum number of clients allowed per cluster, where each client has a unique 128-bit ID.
|
|
25
31
|
/// This impacts the amount of memory allocated at initialization by the server.
|
|
26
32
|
/// This determines the size of the VR client table used to cache replies to clients by client ID.
|
|
@@ -289,8 +295,7 @@ comptime {
|
|
|
289
295
|
/// * replicated storage overhead, since all data files are mirrored,
|
|
290
296
|
/// * the size of the superblock storage zone, and
|
|
291
297
|
/// * the static memory allocation required for tracking LSM forest metadata in memory.
|
|
292
|
-
|
|
293
|
-
pub const size_max = config.cluster.size_max;
|
|
298
|
+
pub const storage_size_max = config.cluster.storage_size_max;
|
|
294
299
|
|
|
295
300
|
/// The unit of read/write access to LSM manifest and LSM table blocks in the block storage zone.
|
|
296
301
|
///
|
|
@@ -298,8 +303,6 @@ pub const size_max = config.cluster.size_max;
|
|
|
298
303
|
/// - A higher block size increases space amplification due to partially-filled blocks.
|
|
299
304
|
pub const block_size = config.cluster.block_size;
|
|
300
305
|
|
|
301
|
-
pub const block_count_max = @divExact(16 * 1024 * 1024 * 1024 * 1024, block_size);
|
|
302
|
-
|
|
303
306
|
comptime {
|
|
304
307
|
assert(block_size % sector_size == 0);
|
|
305
308
|
assert(lsm_table_size_max % sector_size == 0);
|
|
@@ -380,7 +383,7 @@ pub const clock_epoch_max_ms = config.process.clock_epoch_max_ms;
|
|
|
380
383
|
|
|
381
384
|
/// The amount of time to wait for enough accurate samples before synchronizing the clock.
|
|
382
385
|
/// The more samples we can take per remote clock source, the more accurate our estimation becomes.
|
|
383
|
-
/// This impacts cluster startup time as the
|
|
386
|
+
/// This impacts cluster startup time as the primary must first wait for synchronization to complete.
|
|
384
387
|
pub const clock_synchronization_window_min_ms = config.process.clock_synchronization_window_min_ms;
|
|
385
388
|
|
|
386
389
|
/// The amount of time without agreement before the clock window is expired and a new window opened.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const std = @import("std");
|
|
2
2
|
const assert = std.debug.assert;
|
|
3
3
|
|
|
4
|
-
const
|
|
4
|
+
const constants = @import("constants.zig");
|
|
5
5
|
|
|
6
6
|
const tb = @import("tigerbeetle.zig");
|
|
7
7
|
const Account = tb.Account;
|
|
@@ -16,7 +16,7 @@ const Storage = @import("storage.zig").Storage;
|
|
|
16
16
|
const MessagePool = @import("message_pool.zig").MessagePool;
|
|
17
17
|
const MessageBus = @import("message_bus.zig").MessageBusClient;
|
|
18
18
|
const StateMachine = @import("state_machine.zig").StateMachineType(Storage, .{
|
|
19
|
-
.message_body_size_max =
|
|
19
|
+
.message_body_size_max = constants.message_body_size_max,
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
const vsr = @import("vsr.zig");
|
|
@@ -37,7 +37,7 @@ pub fn request(
|
|
|
37
37
|
const allocator = std.heap.page_allocator;
|
|
38
38
|
const client_id = std.crypto.random.int(u128);
|
|
39
39
|
const cluster_id: u32 = 0;
|
|
40
|
-
var addresses = [_]std.net.Address{try std.net.Address.parseIp4("127.0.0.1",
|
|
40
|
+
var addresses = [_]std.net.Address{try std.net.Address.parseIp4("127.0.0.1", constants.port)};
|
|
41
41
|
|
|
42
42
|
var io = try IO.init(32, 0);
|
|
43
43
|
defer io.deinit();
|
|
@@ -74,7 +74,7 @@ pub fn request(
|
|
|
74
74
|
|
|
75
75
|
while (client.request_queue.count > 0) {
|
|
76
76
|
client.tick();
|
|
77
|
-
try io.run_for_ns(
|
|
77
|
+
try io.run_for_ns(constants.tick_ms * std.time.ns_per_ms);
|
|
78
78
|
}
|
|
79
79
|
}
|
|
80
80
|
|
|
@@ -4,7 +4,7 @@ const mem = std.mem;
|
|
|
4
4
|
const assert = std.debug.assert;
|
|
5
5
|
const log = std.log.scoped(.io);
|
|
6
6
|
|
|
7
|
-
const
|
|
7
|
+
const constants = @import("../constants.zig");
|
|
8
8
|
const FIFO = @import("../fifo.zig").FIFO;
|
|
9
9
|
const Time = @import("../time.zig").Time;
|
|
10
10
|
const buffer_limit = @import("../io.zig").buffer_limit;
|
|
@@ -658,8 +658,8 @@ pub const IO = struct {
|
|
|
658
658
|
must_create: bool,
|
|
659
659
|
) !os.fd_t {
|
|
660
660
|
assert(relative_path.len > 0);
|
|
661
|
-
assert(size >=
|
|
662
|
-
assert(size %
|
|
661
|
+
assert(size >= constants.sector_size);
|
|
662
|
+
assert(size % constants.sector_size == 0);
|
|
663
663
|
|
|
664
664
|
// TODO Use O_EXCL when opening as a block device to obtain a mandatory exclusive lock.
|
|
665
665
|
// This is much stronger than an advisory exclusive lock, and is required on some platforms.
|
|
@@ -694,7 +694,7 @@ pub const IO = struct {
|
|
|
694
694
|
|
|
695
695
|
// On darwin assume that Direct I/O is always supported.
|
|
696
696
|
// Use F_NOCACHE to disable the page cache as O_DIRECT doesn't exist.
|
|
697
|
-
if (
|
|
697
|
+
if (constants.direct_io) {
|
|
698
698
|
_ = try os.fcntl(fd, os.F.NOCACHE, 1);
|
|
699
699
|
}
|
|
700
700
|
|
|
@@ -7,7 +7,7 @@ const io_uring_cqe = linux.io_uring_cqe;
|
|
|
7
7
|
const io_uring_sqe = linux.io_uring_sqe;
|
|
8
8
|
const log = std.log.scoped(.io);
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
const constants = @import("../constants.zig");
|
|
11
11
|
const FIFO = @import("../fifo.zig").FIFO;
|
|
12
12
|
const buffer_limit = @import("../io.zig").buffer_limit;
|
|
13
13
|
|
|
@@ -905,8 +905,8 @@ pub const IO = struct {
|
|
|
905
905
|
must_create: bool,
|
|
906
906
|
) !os.fd_t {
|
|
907
907
|
assert(relative_path.len > 0);
|
|
908
|
-
assert(size >=
|
|
909
|
-
assert(size %
|
|
908
|
+
assert(size >= constants.sector_size);
|
|
909
|
+
assert(size % constants.sector_size == 0);
|
|
910
910
|
|
|
911
911
|
// TODO Use O_EXCL when opening as a block device to obtain a mandatory exclusive lock.
|
|
912
912
|
// This is much stronger than an advisory exclusive lock, and is required on some platforms.
|
|
@@ -918,11 +918,11 @@ pub const IO = struct {
|
|
|
918
918
|
if (@hasDecl(os.O, "LARGEFILE")) flags |= os.O.LARGEFILE;
|
|
919
919
|
|
|
920
920
|
var direct_io_supported = false;
|
|
921
|
-
if (
|
|
921
|
+
if (constants.direct_io) {
|
|
922
922
|
direct_io_supported = try fs_supports_direct_io(dir_fd);
|
|
923
923
|
if (direct_io_supported) {
|
|
924
924
|
flags |= os.O.DIRECT;
|
|
925
|
-
} else if (!
|
|
925
|
+
} else if (!constants.direct_io_required) {
|
|
926
926
|
log.warn("file system does not support Direct I/O", .{});
|
|
927
927
|
} else {
|
|
928
928
|
// We require Direct I/O for safety to handle fsync failure correctly, and therefore
|
|
@@ -968,7 +968,7 @@ pub const IO = struct {
|
|
|
968
968
|
log.warn("file system does not support fallocate(), an ENOSPC will panic", .{});
|
|
969
969
|
log.info("allocating by writing to the last sector of the file instead...", .{});
|
|
970
970
|
|
|
971
|
-
const sector_size =
|
|
971
|
+
const sector_size = constants.sector_size;
|
|
972
972
|
const sector: [sector_size]u8 align(sector_size) = [_]u8{0} ** sector_size;
|
|
973
973
|
|
|
974
974
|
// Handle partial writes where the physical sector is less than a logical sector:
|
|
@@ -2,7 +2,7 @@ const std = @import("std");
|
|
|
2
2
|
const os = std.os;
|
|
3
3
|
const assert = std.debug.assert;
|
|
4
4
|
const log = std.log.scoped(.io);
|
|
5
|
-
const
|
|
5
|
+
const constants = @import("../constants.zig");
|
|
6
6
|
|
|
7
7
|
const FIFO = @import("../fifo.zig").FIFO;
|
|
8
8
|
const Time = @import("../time.zig").Time;
|
|
@@ -941,8 +941,8 @@ pub const IO = struct {
|
|
|
941
941
|
must_create: bool,
|
|
942
942
|
) !os.fd_t {
|
|
943
943
|
assert(relative_path.len > 0);
|
|
944
|
-
assert(size >=
|
|
945
|
-
assert(size %
|
|
944
|
+
assert(size >= constants.sector_size);
|
|
945
|
+
assert(size % constants.sector_size == 0);
|
|
946
946
|
|
|
947
947
|
const path_w = try os.windows.sliceToPrefixedFileW(relative_path);
|
|
948
948
|
|
|
@@ -1013,7 +1013,7 @@ pub const IO = struct {
|
|
|
1013
1013
|
log.warn("file system failed to preallocate the file memory", .{});
|
|
1014
1014
|
log.info("allocating by writing to the last sector of the file instead...", .{});
|
|
1015
1015
|
|
|
1016
|
-
const sector_size =
|
|
1016
|
+
const sector_size = constants.sector_size;
|
|
1017
1017
|
const sector: [sector_size]u8 align(sector_size) = [_]u8{0} ** sector_size;
|
|
1018
1018
|
|
|
1019
1019
|
// Handle partial writes where the physical sector is less than a logical sector:
|
|
@@ -38,11 +38,11 @@ const assert = std.debug.assert;
|
|
|
38
38
|
const log = std.log.scoped(.compaction);
|
|
39
39
|
const tracer = @import("../tracer.zig");
|
|
40
40
|
|
|
41
|
-
const
|
|
41
|
+
const constants = @import("../constants.zig");
|
|
42
42
|
|
|
43
43
|
const GridType = @import("grid.zig").GridType;
|
|
44
44
|
const ManifestType = @import("manifest.zig").ManifestType;
|
|
45
|
-
const
|
|
45
|
+
const MergeIteratorType = @import("merge_iterator.zig").MergeIteratorType;
|
|
46
46
|
const TableIteratorType = @import("table_iterator.zig").TableIteratorType;
|
|
47
47
|
const LevelIteratorType = @import("level_iterator.zig").LevelIteratorType;
|
|
48
48
|
|
|
@@ -51,8 +51,6 @@ pub fn CompactionType(
|
|
|
51
51
|
comptime Storage: type,
|
|
52
52
|
comptime IteratorAType: anytype,
|
|
53
53
|
) type {
|
|
54
|
-
const Key = Table.Key;
|
|
55
|
-
const Value = Table.Value;
|
|
56
54
|
const tombstone = Table.tombstone;
|
|
57
55
|
|
|
58
56
|
return struct {
|
|
@@ -73,47 +71,12 @@ pub fn CompactionType(
|
|
|
73
71
|
const IteratorA = IteratorAType(Table, Storage);
|
|
74
72
|
const IteratorB = LevelIteratorType(Table, Storage);
|
|
75
73
|
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
Table.Value,
|
|
81
|
-
Table.key_from_value,
|
|
82
|
-
Table.compare_keys,
|
|
83
|
-
k,
|
|
84
|
-
MergeStreamSelector.peek,
|
|
85
|
-
MergeStreamSelector.pop,
|
|
86
|
-
MergeStreamSelector.precedence,
|
|
74
|
+
const MergeIterator = MergeIteratorType(
|
|
75
|
+
Table,
|
|
76
|
+
IteratorA,
|
|
77
|
+
IteratorB,
|
|
87
78
|
);
|
|
88
79
|
|
|
89
|
-
const MergeStreamSelector = struct {
|
|
90
|
-
fn peek(compaction: *const Compaction, stream_id: u32) error{ Empty, Drained }!Key {
|
|
91
|
-
return switch (stream_id) {
|
|
92
|
-
0 => compaction.iterator_a.peek(),
|
|
93
|
-
1 => compaction.iterator_b.peek(),
|
|
94
|
-
else => unreachable,
|
|
95
|
-
};
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
fn pop(compaction: *Compaction, stream_id: u32) Value {
|
|
99
|
-
return switch (stream_id) {
|
|
100
|
-
0 => compaction.iterator_a.pop(),
|
|
101
|
-
1 => compaction.iterator_b.pop(),
|
|
102
|
-
else => unreachable,
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/// Returns true if stream A has higher precedence than stream B.
|
|
107
|
-
/// This is used to deduplicate values across streams.
|
|
108
|
-
fn precedence(compaction: *const Compaction, stream_a: u32, stream_b: u32) bool {
|
|
109
|
-
_ = compaction;
|
|
110
|
-
assert(stream_a + stream_b == 1);
|
|
111
|
-
|
|
112
|
-
// All tables in iterator_a (stream=0) have a higher precedence.
|
|
113
|
-
return stream_a == 0;
|
|
114
|
-
}
|
|
115
|
-
};
|
|
116
|
-
|
|
117
80
|
pub const Callback = fn (it: *Compaction) void;
|
|
118
81
|
|
|
119
82
|
const Status = enum {
|
|
@@ -226,17 +189,17 @@ pub fn CompactionType(
|
|
|
226
189
|
assert(!compaction.merge_done and compaction.merge_iterator == null);
|
|
227
190
|
assert(compaction.tracer_slot == null);
|
|
228
191
|
|
|
229
|
-
assert(op_min % @divExact(
|
|
192
|
+
assert(op_min % @divExact(constants.lsm_batch_multiple, 2) == 0);
|
|
230
193
|
assert(range.table_count > 0);
|
|
231
194
|
if (table_a) |t| assert(t.visible(op_min));
|
|
232
195
|
|
|
233
|
-
assert(level_b <
|
|
196
|
+
assert(level_b < constants.lsm_levels);
|
|
234
197
|
assert((level_b == 0) == (table_a == null));
|
|
235
198
|
|
|
236
199
|
// Levels may choose to drop tombstones if keys aren't included in the lower levels.
|
|
237
200
|
// This invariant is always true for the last level as it doesn't have any lower ones.
|
|
238
201
|
const drop_tombstones = manifest.compaction_must_drop_tombstones(level_b, range);
|
|
239
|
-
assert(drop_tombstones or level_b <
|
|
202
|
+
assert(drop_tombstones or level_b < constants.lsm_levels - 1);
|
|
240
203
|
|
|
241
204
|
compaction.* = .{
|
|
242
205
|
.tree_name = compaction.tree_name,
|
|
@@ -440,7 +403,10 @@ pub fn CompactionType(
|
|
|
440
403
|
// Create the merge iterator only when we can peek() from the read iterators.
|
|
441
404
|
// This happens after IO for the first reads complete.
|
|
442
405
|
if (compaction.merge_iterator == null) {
|
|
443
|
-
compaction.merge_iterator = MergeIterator.init(
|
|
406
|
+
compaction.merge_iterator = MergeIterator.init(
|
|
407
|
+
&compaction.iterator_a,
|
|
408
|
+
&compaction.iterator_b,
|
|
409
|
+
);
|
|
444
410
|
assert(!compaction.merge_iterator.?.empty());
|
|
445
411
|
}
|
|
446
412
|
|
|
@@ -625,11 +591,11 @@ pub fn CompactionType(
|
|
|
625
591
|
}
|
|
626
592
|
|
|
627
593
|
fn snapshot_max_for_table_input(op_min: u64) u64 {
|
|
628
|
-
assert(op_min % @divExact(
|
|
629
|
-
return op_min + @divExact(
|
|
594
|
+
assert(op_min % @divExact(constants.lsm_batch_multiple, 2) == 0);
|
|
595
|
+
return op_min + @divExact(constants.lsm_batch_multiple, 2) - 1;
|
|
630
596
|
}
|
|
631
597
|
|
|
632
598
|
fn snapshot_min_for_table_output(op_min: u64) u64 {
|
|
633
|
-
assert(op_min % @divExact(
|
|
634
|
-
return op_min + @divExact(
|
|
599
|
+
assert(op_min % @divExact(constants.lsm_batch_multiple, 2) == 0);
|
|
600
|
+
return op_min + @divExact(constants.lsm_batch_multiple, 2);
|
|
635
601
|
}
|
|
@@ -4,11 +4,11 @@ const assert = std.debug.assert;
|
|
|
4
4
|
const math = std.math;
|
|
5
5
|
const mem = std.mem;
|
|
6
6
|
|
|
7
|
-
const
|
|
7
|
+
const constants = @import("../constants.zig");
|
|
8
8
|
const vsr = @import("../vsr.zig");
|
|
9
9
|
|
|
10
10
|
const GridType = @import("grid.zig").GridType;
|
|
11
|
-
const NodePool = @import("node_pool.zig").NodePool(
|
|
11
|
+
const NodePool = @import("node_pool.zig").NodePool(constants.lsm_manifest_node_size, 16);
|
|
12
12
|
|
|
13
13
|
pub fn ForestType(comptime Storage: type, comptime groove_config: anytype) type {
|
|
14
14
|
var groove_fields: []const std.builtin.TypeInfo.StructField = &.{};
|