tigerbeetle-node 0.11.4 → 0.11.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/.client.node.sha256 +1 -1
  2. package/dist/index.d.ts +41 -42
  3. package/dist/index.js +41 -42
  4. package/dist/index.js.map +1 -1
  5. package/package.json +2 -2
  6. package/src/index.ts +0 -1
  7. package/src/node.zig +5 -5
  8. package/src/tigerbeetle/scripts/benchmark.bat +6 -1
  9. package/src/tigerbeetle/scripts/benchmark.sh +1 -1
  10. package/src/tigerbeetle/src/benchmark.zig +4 -4
  11. package/src/tigerbeetle/src/c/tb_client/context.zig +6 -6
  12. package/src/tigerbeetle/src/c/tb_client/echo_client.zig +2 -2
  13. package/src/tigerbeetle/src/c/tb_client/thread.zig +0 -1
  14. package/src/tigerbeetle/src/c/tb_client.h +42 -43
  15. package/src/tigerbeetle/src/c/tb_client.zig +2 -2
  16. package/src/tigerbeetle/src/c/test.zig +8 -8
  17. package/src/tigerbeetle/src/cli.zig +41 -17
  18. package/src/tigerbeetle/src/config.zig +24 -3
  19. package/src/tigerbeetle/src/constants.zig +8 -5
  20. package/src/tigerbeetle/src/demo.zig +4 -4
  21. package/src/tigerbeetle/src/io/darwin.zig +4 -4
  22. package/src/tigerbeetle/src/io/linux.zig +6 -6
  23. package/src/tigerbeetle/src/io/windows.zig +4 -4
  24. package/src/tigerbeetle/src/lsm/compaction.zig +17 -51
  25. package/src/tigerbeetle/src/lsm/forest.zig +2 -2
  26. package/src/tigerbeetle/src/lsm/forest_fuzz.zig +11 -16
  27. package/src/tigerbeetle/src/lsm/grid.zig +5 -5
  28. package/src/tigerbeetle/src/lsm/groove.zig +7 -4
  29. package/src/tigerbeetle/src/lsm/level_iterator.zig +2 -2
  30. package/src/tigerbeetle/src/lsm/manifest.zig +19 -18
  31. package/src/tigerbeetle/src/lsm/manifest_level.zig +3 -2
  32. package/src/tigerbeetle/src/lsm/manifest_log.zig +8 -8
  33. package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig +31 -15
  34. package/src/tigerbeetle/src/lsm/merge_iterator.zig +106 -0
  35. package/src/tigerbeetle/src/lsm/posted_groove.zig +4 -3
  36. package/src/tigerbeetle/src/lsm/segmented_array.zig +1 -0
  37. package/src/tigerbeetle/src/lsm/segmented_array_benchmark.zig +13 -13
  38. package/src/tigerbeetle/src/lsm/set_associative_cache.zig +2 -2
  39. package/src/tigerbeetle/src/lsm/table.zig +33 -19
  40. package/src/tigerbeetle/src/lsm/table_immutable.zig +4 -4
  41. package/src/tigerbeetle/src/lsm/table_iterator.zig +19 -11
  42. package/src/tigerbeetle/src/lsm/table_mutable.zig +52 -18
  43. package/src/tigerbeetle/src/lsm/test.zig +14 -10
  44. package/src/tigerbeetle/src/lsm/tree.zig +48 -45
  45. package/src/tigerbeetle/src/lsm/tree_fuzz.zig +310 -271
  46. package/src/tigerbeetle/src/main.zig +40 -36
  47. package/src/tigerbeetle/src/message_bus.zig +25 -25
  48. package/src/tigerbeetle/src/message_pool.zig +18 -17
  49. package/src/tigerbeetle/src/simulator.zig +28 -83
  50. package/src/tigerbeetle/src/{test/accounting → state_machine}/auditor.zig +9 -9
  51. package/src/tigerbeetle/src/{test/accounting → state_machine}/workload.zig +112 -52
  52. package/src/tigerbeetle/src/state_machine.zig +20 -14
  53. package/src/tigerbeetle/src/storage.zig +12 -12
  54. package/src/tigerbeetle/src/test/cluster.zig +17 -17
  55. package/src/tigerbeetle/src/test/conductor.zig +8 -8
  56. package/src/tigerbeetle/src/test/id.zig +10 -0
  57. package/src/tigerbeetle/src/test/message_bus.zig +0 -2
  58. package/src/tigerbeetle/src/test/network.zig +5 -5
  59. package/src/tigerbeetle/src/test/state_checker.zig +2 -2
  60. package/src/tigerbeetle/src/test/state_machine.zig +151 -46
  61. package/src/tigerbeetle/src/test/storage.zig +54 -51
  62. package/src/tigerbeetle/src/test/storage_checker.zig +3 -3
  63. package/src/tigerbeetle/src/tigerbeetle.zig +0 -1
  64. package/src/tigerbeetle/src/time.zig +0 -1
  65. package/src/tigerbeetle/src/tracer.zig +4 -4
  66. package/src/tigerbeetle/src/unit_tests.zig +2 -2
  67. package/src/tigerbeetle/src/vsr/client.zig +10 -10
  68. package/src/tigerbeetle/src/vsr/clock.zig +11 -10
  69. package/src/tigerbeetle/src/vsr/journal.zig +581 -531
  70. package/src/tigerbeetle/src/vsr/journal_format_fuzz.zig +11 -11
  71. package/src/tigerbeetle/src/vsr/replica.zig +377 -367
  72. package/src/tigerbeetle/src/vsr/replica_format.zig +14 -11
  73. package/src/tigerbeetle/src/vsr/superblock.zig +125 -80
  74. package/src/tigerbeetle/src/vsr/superblock_client_table.zig +9 -9
  75. package/src/tigerbeetle/src/vsr/superblock_free_set.zig +4 -4
  76. package/src/tigerbeetle/src/vsr/superblock_fuzz.zig +11 -6
  77. package/src/tigerbeetle/src/vsr/superblock_manifest.zig +5 -5
  78. package/src/tigerbeetle/src/vsr/superblock_quorums_fuzz.zig +3 -3
  79. package/src/tigerbeetle/src/vsr.zig +25 -25
@@ -6,7 +6,7 @@ const testing = std.testing;
6
6
  const c = @cImport(@cInclude("tb_client.h"));
7
7
 
8
8
  const util = @import("../util.zig");
9
- const config = @import("../constants.zig");
9
+ const constants = @import("../constants.zig");
10
10
  const Packet = @import("tb_client/packet.zig").Packet;
11
11
 
12
12
  const Mutex = std.Thread.Mutex;
@@ -91,10 +91,10 @@ const Completion = struct {
91
91
  // 3. the data marshaling is correct, and exactly the same data sent was received back.
92
92
  test "c_client echo" {
93
93
  // Using the create_accounts operation for this test.
94
- const RequestContext = RequestContextType(config.message_body_size_max);
94
+ const RequestContext = RequestContextType(constants.message_body_size_max);
95
95
  const create_accounts_operation: u8 = c.TB_OPERATION_CREATE_ACCOUNTS;
96
96
  const event_size = @sizeOf(c.tb_account_t);
97
- const event_request_max = @divFloor(config.message_body_size_max, event_size);
97
+ const event_request_max = @divFloor(constants.message_body_size_max, event_size);
98
98
 
99
99
  // Initializing an echo client for testing purposes.
100
100
  // We ensure that the retry mechanism is being tested
@@ -103,7 +103,7 @@ test "c_client echo" {
103
103
  var tb_packet_list: c.tb_packet_list_t = undefined;
104
104
  const cluster_id = 0;
105
105
  const address = "3000";
106
- const packets_count: u32 = config.client_request_queue_max * 2;
106
+ const packets_count: u32 = constants.client_request_queue_max * 2;
107
107
  const tb_context: usize = 42;
108
108
  const result = c.tb_client_init_echo(
109
109
  &tb_client,
@@ -221,7 +221,7 @@ test "c_client tb_status" {
221
221
  // More addresses thant "replicas_max" should return "TB_STATUS_ADDRESS_LIMIT_EXCEEDED":
222
222
  try assert_status(
223
223
  1,
224
- ("3000," ** config.replicas_max) ++ "3001",
224
+ ("3000," ** constants.replicas_max) ++ "3001",
225
225
  c.TB_STATUS_ADDRESS_LIMIT_EXCEEDED,
226
226
  );
227
227
 
@@ -234,7 +234,7 @@ test "c_client tb_status" {
234
234
 
235
235
  // Asserts the validation rules associated with the "TB_PACKET_STATUS" enum.
236
236
  test "c_client tb_packet_status" {
237
- const RequestContext = RequestContextType(config.message_body_size_max);
237
+ const RequestContext = RequestContextType(constants.message_body_size_max);
238
238
 
239
239
  var tb_client: c.tb_client_t = undefined;
240
240
  var tb_packet_list: c.tb_packet_list_t = undefined;
@@ -299,12 +299,12 @@ test "c_client tb_packet_status" {
299
299
 
300
300
  var packet_list = @ptrCast(*Packet.List, &tb_packet_list);
301
301
 
302
- // Messages larger than config.message_body_size_max should return "too_much_data":
302
+ // Messages larger than constants.message_body_size_max should return "too_much_data":
303
303
  try assert_result(
304
304
  tb_client,
305
305
  packet_list,
306
306
  c.TB_OPERATION_CREATE_TRANSFERS,
307
- config.message_body_size_max + @sizeOf(c.tb_transfer_t),
307
+ constants.message_body_size_max + @sizeOf(c.tb_transfer_t),
308
308
  c.TB_PACKET_TOO_MUCH_DATA,
309
309
  );
310
310
 
@@ -7,12 +7,13 @@ const meta = std.meta;
7
7
  const net = std.net;
8
8
  const os = std.os;
9
9
 
10
- const config = @import("constants.zig");
10
+ const constants = @import("constants.zig");
11
11
  const tigerbeetle = @import("tigerbeetle.zig");
12
12
  const vsr = @import("vsr.zig");
13
13
  const IO = @import("io.zig").IO;
14
+ const data_file_size_min = @import("vsr/superblock.zig").data_file_size_min;
14
15
 
15
- // TODO Document --cache-accounts, --cache-transfers, --cache-transfers-posted
16
+ // TODO Document --cache-accounts, --cache-transfers, --cache-transfers-posted, --limit-storage
16
17
  const usage = fmt.comptimePrint(
17
18
  \\Usage:
18
19
  \\
@@ -72,25 +73,28 @@ const usage = fmt.comptimePrint(
72
73
  \\ tigerbeetle version --verbose
73
74
  \\
74
75
  , .{
75
- .default_address = config.address,
76
- .default_port = config.port,
76
+ .default_address = constants.address,
77
+ .default_port = constants.port,
77
78
  });
78
79
 
79
80
  pub const Command = union(enum) {
80
- format: struct {
81
- args_allocated: std.ArrayList([:0]const u8),
82
- cluster: u32,
83
- replica: u8,
84
- path: [:0]const u8,
85
- },
86
- start: struct {
81
+ pub const Start = struct {
87
82
  args_allocated: std.ArrayList([:0]const u8),
88
83
  addresses: []net.Address,
89
84
  cache_accounts: u32,
90
85
  cache_transfers: u32,
91
86
  cache_transfers_posted: u32,
87
+ storage_size_limit: u64,
88
+ path: [:0]const u8,
89
+ };
90
+
91
+ format: struct {
92
+ args_allocated: std.ArrayList([:0]const u8),
93
+ cluster: u32,
94
+ replica: u8,
92
95
  path: [:0]const u8,
93
96
  },
97
+ start: Start,
94
98
  version: struct {
95
99
  verbose: bool,
96
100
  },
@@ -117,6 +121,7 @@ pub fn parse_args(allocator: std.mem.Allocator) !Command {
117
121
  var cache_accounts: ?[]const u8 = null;
118
122
  var cache_transfers: ?[]const u8 = null;
119
123
  var cache_transfers_posted: ?[]const u8 = null;
124
+ var storage_size_limit: ?[]const u8 = null;
120
125
  var verbose: ?bool = null;
121
126
 
122
127
  var args = try std.process.argsWithAllocator(allocator);
@@ -163,6 +168,9 @@ pub fn parse_args(allocator: std.mem.Allocator) !Command {
163
168
  } else if (mem.startsWith(u8, arg, "--cache-transfers-posted")) {
164
169
  if (command != .start) fatal("--cache-transfers-posted: supported only by 'start' command", .{});
165
170
  cache_transfers_posted = parse_flag("--cache-transfers-posted", arg);
171
+ } else if (mem.startsWith(u8, arg, "--limit-storage")) {
172
+ if (command != .start) fatal("--limit-storage: supported only by 'start' command", .{});
173
+ storage_size_limit = parse_flag("--limit-storage", arg);
166
174
  } else if (mem.eql(u8, arg, "--verbose")) {
167
175
  if (command != .version) fatal("--verbose: supported only by 'version' command", .{});
168
176
  verbose = true;
@@ -206,18 +214,19 @@ pub fn parse_args(allocator: std.mem.Allocator) !Command {
206
214
  .cache_accounts = parse_size_to_count(
207
215
  tigerbeetle.Account,
208
216
  cache_accounts,
209
- config.cache_accounts_max,
217
+ constants.cache_accounts_max,
210
218
  ),
211
219
  .cache_transfers = parse_size_to_count(
212
220
  tigerbeetle.Transfer,
213
221
  cache_transfers,
214
- config.cache_transfers_max,
222
+ constants.cache_transfers_max,
215
223
  ),
216
224
  .cache_transfers_posted = parse_size_to_count(
217
225
  u256, // TODO(#264): Use actual type here, once exposed.
218
226
  cache_transfers_posted,
219
- config.cache_transfers_posted_max,
227
+ constants.cache_transfers_posted_max,
220
228
  ),
229
+ .storage_size_limit = parse_storage_size(storage_size_limit),
221
230
  .path = path orelse fatal("required: <path>", .{}),
222
231
  },
223
232
  };
@@ -255,11 +264,11 @@ fn parse_cluster(raw_cluster: []const u8) u32 {
255
264
 
256
265
  /// Parse and allocate the addresses returning a slice into that array.
257
266
  fn parse_addresses(allocator: std.mem.Allocator, raw_addresses: []const u8) []net.Address {
258
- return vsr.parse_addresses(allocator, raw_addresses, config.replicas_max) catch |err| switch (err) {
267
+ return vsr.parse_addresses(allocator, raw_addresses, constants.replicas_max) catch |err| switch (err) {
259
268
  error.AddressHasTrailingComma => fatal("--addresses: invalid trailing comma", .{}),
260
269
  error.AddressLimitExceeded => {
261
270
  fatal("--addresses: too many addresses, at most {d} are allowed", .{
262
- config.replicas_max,
271
+ constants.replicas_max,
263
272
  });
264
273
  },
265
274
  error.AddressHasMoreThanOneColon => {
@@ -272,6 +281,21 @@ fn parse_addresses(allocator: std.mem.Allocator, raw_addresses: []const u8) []ne
272
281
  };
273
282
  }
274
283
 
284
+ fn parse_storage_size(size_string: ?[]const u8) u64 {
285
+ const size_min = data_file_size_min;
286
+ const size_max = constants.storage_size_max;
287
+ const size = if (size_string) |s| parse_size(s) else size_max;
288
+ if (size > size_max) fatal("storage size {} exceeds maximum: {}", .{ size, size_max });
289
+ if (size < size_min) fatal("storage size {} is below minimum: {}", .{ size, size_min });
290
+ if (size % constants.sector_size != 0) {
291
+ fatal("size value {} must be a multiple of sector size ({})", .{
292
+ size,
293
+ constants.sector_size,
294
+ });
295
+ }
296
+ return size;
297
+ }
298
+
275
299
  fn parse_size(string: []const u8) u64 {
276
300
  var value = mem.trim(u8, string, " ");
277
301
 
@@ -366,7 +390,7 @@ fn parse_size_to_count(comptime T: type, string_opt: ?[]const u8, comptime defau
366
390
  }
367
391
 
368
392
  fn parse_replica(raw_replica: []const u8) u8 {
369
- comptime assert(config.replicas_max <= std.math.maxInt(u8));
393
+ comptime assert(constants.replicas_max <= std.math.maxInt(u8));
370
394
  const replica = fmt.parseUnsigned(u8, raw_replica, 10) catch |err| switch (err) {
371
395
  error.Overflow => fatal("--replica: value exceeds an 8-bit unsigned integer", .{}),
372
396
  error.InvalidCharacter => fatal("--replica: value contains an invalid character", .{}),
@@ -21,7 +21,7 @@ pub const Config = struct {
21
21
  /// - Client configs need not equal replica configs.
22
22
  /// - Replica configs can change between restarts.
23
23
  ///
24
- /// Fields are documented within the top-level module body.
24
+ /// Fields are documented within constants.zig.
25
25
  const ConfigProcess = struct {
26
26
  log_level: std.log.Level = .info,
27
27
  tracer_backend: TracerBackend = .none,
@@ -64,7 +64,7 @@ const ConfigProcess = struct {
64
64
  /// over the cluster lifetime.
65
65
  /// - The storage formats generated by different ConfigClusters are incompatible.
66
66
  ///
67
- /// Fields are documented within the top-level module body.
67
+ /// Fields are documented within constants.zig.
68
68
  const ConfigCluster = struct {
69
69
  cache_line_size: comptime_int = 64,
70
70
  clients_max: usize,
@@ -72,7 +72,7 @@ const ConfigCluster = struct {
72
72
  journal_slot_count: usize = 1024,
73
73
  message_size_max: usize = 1 * 1024 * 1024,
74
74
  superblock_copies: comptime_int = 4,
75
- size_max: usize = 16 * 1024 * 1024 * 1024 * 1024,
75
+ storage_size_max: u64 = 16 * 1024 * 1024 * 1024 * 1024,
76
76
  block_size: comptime_int = 64 * 1024,
77
77
  lsm_levels: u7 = 7,
78
78
  lsm_growth_factor: u32 = 8,
@@ -80,6 +80,7 @@ const ConfigCluster = struct {
80
80
  lsm_batch_multiple: comptime_int = 4,
81
81
  lsm_snapshots_max: usize = 32,
82
82
  lsm_value_to_key_layout_ratio_min: comptime_int = 16,
83
+ state_machine: StateMachine = .accounting,
83
84
 
84
85
  /// The WAL requires at least two sectors of redundant headers — otherwise we could lose them all to
85
86
  /// a single torn write. A replica needs at least one valid redundant header to determine an
@@ -102,6 +103,13 @@ const ConfigCluster = struct {
102
103
  }
103
104
  };
104
105
 
106
+ pub const ConfigBase = enum {
107
+ production,
108
+ development,
109
+ test_min,
110
+ default,
111
+ };
112
+
105
113
  pub const TracerBackend = enum {
106
114
  none,
107
115
  // Writes to a file (./tracer.json) which can be uploaded to https://ui.perfetto.dev/
@@ -110,6 +118,11 @@ pub const TracerBackend = enum {
110
118
  tracy,
111
119
  };
112
120
 
121
+ pub const StateMachine = enum {
122
+ accounting,
123
+ testing,
124
+ };
125
+
113
126
  pub const configs = struct {
114
127
  /// A good default config for production.
115
128
  pub const default_production = Config{
@@ -156,6 +169,7 @@ pub const configs = struct {
156
169
  .clients_max = 4,
157
170
  .journal_slot_count = Config.Cluster.journal_slot_count_min,
158
171
  .message_size_max = Config.Cluster.message_size_max_min(2),
172
+ .storage_size_max = 1024 * 1024 * 1024,
159
173
 
160
174
  .block_size = sector_size,
161
175
  .lsm_growth_factor = 4,
@@ -182,6 +196,13 @@ pub const configs = struct {
182
196
  .test_min => test_min,
183
197
  };
184
198
 
199
+ base.cluster.state_machine = if (@hasDecl(root, "decode_events"))
200
+ // TODO(DJ) This is a hack to work around the absense of tigerbeetle_build_options.
201
+ // This should be removed once the node client is built using `zig build`.
202
+ .accounting
203
+ else
204
+ @intToEnum(StateMachine, @enumToInt(build_options.config_cluster_state_machine));
205
+
185
206
  // TODO Use additional build options to overwrite other fields.
186
207
  base.process.tracer_backend = if (@hasDecl(root, "tracer_backend"))
187
208
  // TODO(jamii)
@@ -21,6 +21,12 @@ pub const tracer_backend = config.process.tracer_backend;
21
21
  /// The maximum number of replicas allowed in a cluster.
22
22
  pub const replicas_max = 6;
23
23
 
24
+ pub const state_machine = config.cluster.state_machine;
25
+ pub const StateMachineType = switch (config.cluster.state_machine) {
26
+ .accounting => @import("state_machine.zig").StateMachineType,
27
+ .testing => @import("test/state_machine.zig").StateMachineType,
28
+ };
29
+
24
30
  /// The maximum number of clients allowed per cluster, where each client has a unique 128-bit ID.
25
31
  /// This impacts the amount of memory allocated at initialization by the server.
26
32
  /// This determines the size of the VR client table used to cache replies to clients by client ID.
@@ -289,8 +295,7 @@ comptime {
289
295
  /// * replicated storage overhead, since all data files are mirrored,
290
296
  /// * the size of the superblock storage zone, and
291
297
  /// * the static memory allocation required for tracking LSM forest metadata in memory.
292
- // TODO Remove, now that we have block_count_max.
293
- pub const size_max = config.cluster.size_max;
298
+ pub const storage_size_max = config.cluster.storage_size_max;
294
299
 
295
300
  /// The unit of read/write access to LSM manifest and LSM table blocks in the block storage zone.
296
301
  ///
@@ -298,8 +303,6 @@ pub const size_max = config.cluster.size_max;
298
303
  /// - A higher block size increases space amplification due to partially-filled blocks.
299
304
  pub const block_size = config.cluster.block_size;
300
305
 
301
- pub const block_count_max = @divExact(16 * 1024 * 1024 * 1024 * 1024, block_size);
302
-
303
306
  comptime {
304
307
  assert(block_size % sector_size == 0);
305
308
  assert(lsm_table_size_max % sector_size == 0);
@@ -380,7 +383,7 @@ pub const clock_epoch_max_ms = config.process.clock_epoch_max_ms;
380
383
 
381
384
  /// The amount of time to wait for enough accurate samples before synchronizing the clock.
382
385
  /// The more samples we can take per remote clock source, the more accurate our estimation becomes.
383
- /// This impacts cluster startup time as the leader must first wait for synchronization to complete.
386
+ /// This impacts cluster startup time as the primary must first wait for synchronization to complete.
384
387
  pub const clock_synchronization_window_min_ms = config.process.clock_synchronization_window_min_ms;
385
388
 
386
389
  /// The amount of time without agreement before the clock window is expired and a new window opened.
@@ -1,7 +1,7 @@
1
1
  const std = @import("std");
2
2
  const assert = std.debug.assert;
3
3
 
4
- const config = @import("constants.zig");
4
+ const constants = @import("constants.zig");
5
5
 
6
6
  const tb = @import("tigerbeetle.zig");
7
7
  const Account = tb.Account;
@@ -16,7 +16,7 @@ const Storage = @import("storage.zig").Storage;
16
16
  const MessagePool = @import("message_pool.zig").MessagePool;
17
17
  const MessageBus = @import("message_bus.zig").MessageBusClient;
18
18
  const StateMachine = @import("state_machine.zig").StateMachineType(Storage, .{
19
- .message_body_size_max = config.message_body_size_max,
19
+ .message_body_size_max = constants.message_body_size_max,
20
20
  });
21
21
 
22
22
  const vsr = @import("vsr.zig");
@@ -37,7 +37,7 @@ pub fn request(
37
37
  const allocator = std.heap.page_allocator;
38
38
  const client_id = std.crypto.random.int(u128);
39
39
  const cluster_id: u32 = 0;
40
- var addresses = [_]std.net.Address{try std.net.Address.parseIp4("127.0.0.1", config.port)};
40
+ var addresses = [_]std.net.Address{try std.net.Address.parseIp4("127.0.0.1", constants.port)};
41
41
 
42
42
  var io = try IO.init(32, 0);
43
43
  defer io.deinit();
@@ -74,7 +74,7 @@ pub fn request(
74
74
 
75
75
  while (client.request_queue.count > 0) {
76
76
  client.tick();
77
- try io.run_for_ns(config.tick_ms * std.time.ns_per_ms);
77
+ try io.run_for_ns(constants.tick_ms * std.time.ns_per_ms);
78
78
  }
79
79
  }
80
80
 
@@ -4,7 +4,7 @@ const mem = std.mem;
4
4
  const assert = std.debug.assert;
5
5
  const log = std.log.scoped(.io);
6
6
 
7
- const config = @import("../constants.zig");
7
+ const constants = @import("../constants.zig");
8
8
  const FIFO = @import("../fifo.zig").FIFO;
9
9
  const Time = @import("../time.zig").Time;
10
10
  const buffer_limit = @import("../io.zig").buffer_limit;
@@ -658,8 +658,8 @@ pub const IO = struct {
658
658
  must_create: bool,
659
659
  ) !os.fd_t {
660
660
  assert(relative_path.len > 0);
661
- assert(size >= config.sector_size);
662
- assert(size % config.sector_size == 0);
661
+ assert(size >= constants.sector_size);
662
+ assert(size % constants.sector_size == 0);
663
663
 
664
664
  // TODO Use O_EXCL when opening as a block device to obtain a mandatory exclusive lock.
665
665
  // This is much stronger than an advisory exclusive lock, and is required on some platforms.
@@ -694,7 +694,7 @@ pub const IO = struct {
694
694
 
695
695
  // On darwin assume that Direct I/O is always supported.
696
696
  // Use F_NOCACHE to disable the page cache as O_DIRECT doesn't exist.
697
- if (config.direct_io) {
697
+ if (constants.direct_io) {
698
698
  _ = try os.fcntl(fd, os.F.NOCACHE, 1);
699
699
  }
700
700
 
@@ -7,7 +7,7 @@ const io_uring_cqe = linux.io_uring_cqe;
7
7
  const io_uring_sqe = linux.io_uring_sqe;
8
8
  const log = std.log.scoped(.io);
9
9
 
10
- const config = @import("../constants.zig");
10
+ const constants = @import("../constants.zig");
11
11
  const FIFO = @import("../fifo.zig").FIFO;
12
12
  const buffer_limit = @import("../io.zig").buffer_limit;
13
13
 
@@ -905,8 +905,8 @@ pub const IO = struct {
905
905
  must_create: bool,
906
906
  ) !os.fd_t {
907
907
  assert(relative_path.len > 0);
908
- assert(size >= config.sector_size);
909
- assert(size % config.sector_size == 0);
908
+ assert(size >= constants.sector_size);
909
+ assert(size % constants.sector_size == 0);
910
910
 
911
911
  // TODO Use O_EXCL when opening as a block device to obtain a mandatory exclusive lock.
912
912
  // This is much stronger than an advisory exclusive lock, and is required on some platforms.
@@ -918,11 +918,11 @@ pub const IO = struct {
918
918
  if (@hasDecl(os.O, "LARGEFILE")) flags |= os.O.LARGEFILE;
919
919
 
920
920
  var direct_io_supported = false;
921
- if (config.direct_io) {
921
+ if (constants.direct_io) {
922
922
  direct_io_supported = try fs_supports_direct_io(dir_fd);
923
923
  if (direct_io_supported) {
924
924
  flags |= os.O.DIRECT;
925
- } else if (!config.direct_io_required) {
925
+ } else if (!constants.direct_io_required) {
926
926
  log.warn("file system does not support Direct I/O", .{});
927
927
  } else {
928
928
  // We require Direct I/O for safety to handle fsync failure correctly, and therefore
@@ -968,7 +968,7 @@ pub const IO = struct {
968
968
  log.warn("file system does not support fallocate(), an ENOSPC will panic", .{});
969
969
  log.info("allocating by writing to the last sector of the file instead...", .{});
970
970
 
971
- const sector_size = config.sector_size;
971
+ const sector_size = constants.sector_size;
972
972
  const sector: [sector_size]u8 align(sector_size) = [_]u8{0} ** sector_size;
973
973
 
974
974
  // Handle partial writes where the physical sector is less than a logical sector:
@@ -2,7 +2,7 @@ const std = @import("std");
2
2
  const os = std.os;
3
3
  const assert = std.debug.assert;
4
4
  const log = std.log.scoped(.io);
5
- const config = @import("../constants.zig");
5
+ const constants = @import("../constants.zig");
6
6
 
7
7
  const FIFO = @import("../fifo.zig").FIFO;
8
8
  const Time = @import("../time.zig").Time;
@@ -941,8 +941,8 @@ pub const IO = struct {
941
941
  must_create: bool,
942
942
  ) !os.fd_t {
943
943
  assert(relative_path.len > 0);
944
- assert(size >= config.sector_size);
945
- assert(size % config.sector_size == 0);
944
+ assert(size >= constants.sector_size);
945
+ assert(size % constants.sector_size == 0);
946
946
 
947
947
  const path_w = try os.windows.sliceToPrefixedFileW(relative_path);
948
948
 
@@ -1013,7 +1013,7 @@ pub const IO = struct {
1013
1013
  log.warn("file system failed to preallocate the file memory", .{});
1014
1014
  log.info("allocating by writing to the last sector of the file instead...", .{});
1015
1015
 
1016
- const sector_size = config.sector_size;
1016
+ const sector_size = constants.sector_size;
1017
1017
  const sector: [sector_size]u8 align(sector_size) = [_]u8{0} ** sector_size;
1018
1018
 
1019
1019
  // Handle partial writes where the physical sector is less than a logical sector:
@@ -38,11 +38,11 @@ const assert = std.debug.assert;
38
38
  const log = std.log.scoped(.compaction);
39
39
  const tracer = @import("../tracer.zig");
40
40
 
41
- const config = @import("../constants.zig");
41
+ const constants = @import("../constants.zig");
42
42
 
43
43
  const GridType = @import("grid.zig").GridType;
44
44
  const ManifestType = @import("manifest.zig").ManifestType;
45
- const KWayMergeIterator = @import("k_way_merge.zig").KWayMergeIterator;
45
+ const MergeIteratorType = @import("merge_iterator.zig").MergeIteratorType;
46
46
  const TableIteratorType = @import("table_iterator.zig").TableIteratorType;
47
47
  const LevelIteratorType = @import("level_iterator.zig").LevelIteratorType;
48
48
 
@@ -51,8 +51,6 @@ pub fn CompactionType(
51
51
  comptime Storage: type,
52
52
  comptime IteratorAType: anytype,
53
53
  ) type {
54
- const Key = Table.Key;
55
- const Value = Table.Value;
56
54
  const tombstone = Table.tombstone;
57
55
 
58
56
  return struct {
@@ -73,47 +71,12 @@ pub fn CompactionType(
73
71
  const IteratorA = IteratorAType(Table, Storage);
74
72
  const IteratorB = LevelIteratorType(Table, Storage);
75
73
 
76
- const k = 2;
77
- const MergeIterator = KWayMergeIterator(
78
- Compaction,
79
- Table.Key,
80
- Table.Value,
81
- Table.key_from_value,
82
- Table.compare_keys,
83
- k,
84
- MergeStreamSelector.peek,
85
- MergeStreamSelector.pop,
86
- MergeStreamSelector.precedence,
74
+ const MergeIterator = MergeIteratorType(
75
+ Table,
76
+ IteratorA,
77
+ IteratorB,
87
78
  );
88
79
 
89
- const MergeStreamSelector = struct {
90
- fn peek(compaction: *const Compaction, stream_id: u32) error{ Empty, Drained }!Key {
91
- return switch (stream_id) {
92
- 0 => compaction.iterator_a.peek(),
93
- 1 => compaction.iterator_b.peek(),
94
- else => unreachable,
95
- };
96
- }
97
-
98
- fn pop(compaction: *Compaction, stream_id: u32) Value {
99
- return switch (stream_id) {
100
- 0 => compaction.iterator_a.pop(),
101
- 1 => compaction.iterator_b.pop(),
102
- else => unreachable,
103
- };
104
- }
105
-
106
- /// Returns true if stream A has higher precedence than stream B.
107
- /// This is used to deduplicate values across streams.
108
- fn precedence(compaction: *const Compaction, stream_a: u32, stream_b: u32) bool {
109
- _ = compaction;
110
- assert(stream_a + stream_b == 1);
111
-
112
- // All tables in iterator_a (stream=0) have a higher precedence.
113
- return stream_a == 0;
114
- }
115
- };
116
-
117
80
  pub const Callback = fn (it: *Compaction) void;
118
81
 
119
82
  const Status = enum {
@@ -226,17 +189,17 @@ pub fn CompactionType(
226
189
  assert(!compaction.merge_done and compaction.merge_iterator == null);
227
190
  assert(compaction.tracer_slot == null);
228
191
 
229
- assert(op_min % @divExact(config.lsm_batch_multiple, 2) == 0);
192
+ assert(op_min % @divExact(constants.lsm_batch_multiple, 2) == 0);
230
193
  assert(range.table_count > 0);
231
194
  if (table_a) |t| assert(t.visible(op_min));
232
195
 
233
- assert(level_b < config.lsm_levels);
196
+ assert(level_b < constants.lsm_levels);
234
197
  assert((level_b == 0) == (table_a == null));
235
198
 
236
199
  // Levels may choose to drop tombstones if keys aren't included in the lower levels.
237
200
  // This invariant is always true for the last level as it doesn't have any lower ones.
238
201
  const drop_tombstones = manifest.compaction_must_drop_tombstones(level_b, range);
239
- assert(drop_tombstones or level_b < config.lsm_levels - 1);
202
+ assert(drop_tombstones or level_b < constants.lsm_levels - 1);
240
203
 
241
204
  compaction.* = .{
242
205
  .tree_name = compaction.tree_name,
@@ -440,7 +403,10 @@ pub fn CompactionType(
440
403
  // Create the merge iterator only when we can peek() from the read iterators.
441
404
  // This happens after IO for the first reads complete.
442
405
  if (compaction.merge_iterator == null) {
443
- compaction.merge_iterator = MergeIterator.init(compaction, k, .ascending);
406
+ compaction.merge_iterator = MergeIterator.init(
407
+ &compaction.iterator_a,
408
+ &compaction.iterator_b,
409
+ );
444
410
  assert(!compaction.merge_iterator.?.empty());
445
411
  }
446
412
 
@@ -625,11 +591,11 @@ pub fn CompactionType(
625
591
  }
626
592
 
627
593
  fn snapshot_max_for_table_input(op_min: u64) u64 {
628
- assert(op_min % @divExact(config.lsm_batch_multiple, 2) == 0);
629
- return op_min + @divExact(config.lsm_batch_multiple, 2) - 1;
594
+ assert(op_min % @divExact(constants.lsm_batch_multiple, 2) == 0);
595
+ return op_min + @divExact(constants.lsm_batch_multiple, 2) - 1;
630
596
  }
631
597
 
632
598
  fn snapshot_min_for_table_output(op_min: u64) u64 {
633
- assert(op_min % @divExact(config.lsm_batch_multiple, 2) == 0);
634
- return op_min + @divExact(config.lsm_batch_multiple, 2);
599
+ assert(op_min % @divExact(constants.lsm_batch_multiple, 2) == 0);
600
+ return op_min + @divExact(constants.lsm_batch_multiple, 2);
635
601
  }
@@ -4,11 +4,11 @@ const assert = std.debug.assert;
4
4
  const math = std.math;
5
5
  const mem = std.mem;
6
6
 
7
- const config = @import("../constants.zig");
7
+ const constants = @import("../constants.zig");
8
8
  const vsr = @import("../vsr.zig");
9
9
 
10
10
  const GridType = @import("grid.zig").GridType;
11
- const NodePool = @import("node_pool.zig").NodePool(config.lsm_manifest_node_size, 16);
11
+ const NodePool = @import("node_pool.zig").NodePool(constants.lsm_manifest_node_size, 16);
12
12
 
13
13
  pub fn ForestType(comptime Storage: type, comptime groove_config: anytype) type {
14
14
  var groove_fields: []const std.builtin.TypeInfo.StructField = &.{};