tigerbeetle-node 0.11.0 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/.client.node.sha256 +1 -0
  2. package/package.json +5 -3
  3. package/src/tigerbeetle/scripts/fuzz_loop.sh +1 -1
  4. package/src/tigerbeetle/scripts/pre-commit.sh +2 -2
  5. package/src/tigerbeetle/scripts/validate_docs.sh +17 -0
  6. package/src/tigerbeetle/src/benchmark.zig +25 -11
  7. package/src/tigerbeetle/src/c/tb_client/context.zig +248 -47
  8. package/src/tigerbeetle/src/c/tb_client/echo_client.zig +108 -0
  9. package/src/tigerbeetle/src/c/tb_client/packet.zig +2 -2
  10. package/src/tigerbeetle/src/c/tb_client/signal.zig +2 -4
  11. package/src/tigerbeetle/src/c/tb_client/thread.zig +17 -256
  12. package/src/tigerbeetle/src/c/tb_client.h +18 -4
  13. package/src/tigerbeetle/src/c/tb_client.zig +88 -26
  14. package/src/tigerbeetle/src/c/tb_client_header_test.zig +135 -0
  15. package/src/tigerbeetle/src/c/test.zig +371 -1
  16. package/src/tigerbeetle/src/cli.zig +90 -18
  17. package/src/tigerbeetle/src/config.zig +12 -4
  18. package/src/tigerbeetle/src/demo.zig +2 -1
  19. package/src/tigerbeetle/src/demo_01_create_accounts.zig +1 -1
  20. package/src/tigerbeetle/src/demo_03_create_transfers.zig +13 -0
  21. package/src/tigerbeetle/src/ewah.zig +11 -33
  22. package/src/tigerbeetle/src/ewah_benchmark.zig +8 -9
  23. package/src/tigerbeetle/src/lsm/README.md +97 -3
  24. package/src/tigerbeetle/src/lsm/compaction.zig +32 -7
  25. package/src/tigerbeetle/src/{eytzinger_benchmark.zig → lsm/eytzinger_benchmark.zig} +34 -21
  26. package/src/tigerbeetle/src/lsm/forest_fuzz.zig +34 -32
  27. package/src/tigerbeetle/src/lsm/grid.zig +39 -21
  28. package/src/tigerbeetle/src/lsm/groove.zig +1 -0
  29. package/src/tigerbeetle/src/lsm/k_way_merge.zig +3 -3
  30. package/src/tigerbeetle/src/lsm/level_iterator.zig +1 -1
  31. package/src/tigerbeetle/src/lsm/manifest.zig +13 -0
  32. package/src/tigerbeetle/src/lsm/manifest_level.zig +0 -49
  33. package/src/tigerbeetle/src/lsm/manifest_log.zig +173 -335
  34. package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig +665 -0
  35. package/src/tigerbeetle/src/lsm/node_pool.zig +4 -0
  36. package/src/tigerbeetle/src/lsm/posted_groove.zig +1 -0
  37. package/src/tigerbeetle/src/lsm/segmented_array.zig +24 -15
  38. package/src/tigerbeetle/src/lsm/table.zig +32 -20
  39. package/src/tigerbeetle/src/lsm/table_immutable.zig +1 -1
  40. package/src/tigerbeetle/src/lsm/table_iterator.zig +4 -5
  41. package/src/tigerbeetle/src/lsm/test.zig +13 -2
  42. package/src/tigerbeetle/src/lsm/tree.zig +45 -7
  43. package/src/tigerbeetle/src/lsm/tree_fuzz.zig +36 -32
  44. package/src/tigerbeetle/src/main.zig +69 -13
  45. package/src/tigerbeetle/src/message_bus.zig +18 -7
  46. package/src/tigerbeetle/src/message_pool.zig +8 -2
  47. package/src/tigerbeetle/src/ring_buffer.zig +7 -3
  48. package/src/tigerbeetle/src/simulator.zig +38 -11
  49. package/src/tigerbeetle/src/state_machine.zig +48 -23
  50. package/src/tigerbeetle/src/test/accounting/workload.zig +9 -5
  51. package/src/tigerbeetle/src/test/cluster.zig +15 -33
  52. package/src/tigerbeetle/src/test/conductor.zig +2 -1
  53. package/src/tigerbeetle/src/test/network.zig +45 -19
  54. package/src/tigerbeetle/src/test/packet_simulator.zig +40 -29
  55. package/src/tigerbeetle/src/test/state_checker.zig +5 -7
  56. package/src/tigerbeetle/src/test/storage.zig +453 -110
  57. package/src/tigerbeetle/src/test/storage_checker.zig +204 -0
  58. package/src/tigerbeetle/src/tigerbeetle.zig +1 -0
  59. package/src/tigerbeetle/src/unit_tests.zig +7 -1
  60. package/src/tigerbeetle/src/util.zig +97 -11
  61. package/src/tigerbeetle/src/vopr.zig +2 -1
  62. package/src/tigerbeetle/src/vsr/client.zig +8 -3
  63. package/src/tigerbeetle/src/vsr/journal.zig +280 -202
  64. package/src/tigerbeetle/src/vsr/replica.zig +169 -31
  65. package/src/tigerbeetle/src/vsr/superblock.zig +356 -629
  66. package/src/tigerbeetle/src/vsr/superblock_client_table.zig +7 -6
  67. package/src/tigerbeetle/src/vsr/superblock_free_set.zig +414 -151
  68. package/src/tigerbeetle/src/vsr/superblock_free_set_fuzz.zig +332 -0
  69. package/src/tigerbeetle/src/vsr/superblock_fuzz.zig +349 -0
  70. package/src/tigerbeetle/src/vsr/superblock_manifest.zig +44 -9
  71. package/src/tigerbeetle/src/vsr/superblock_quorums.zig +394 -0
  72. package/src/tigerbeetle/src/vsr/superblock_quorums_fuzz.zig +312 -0
  73. package/src/tigerbeetle/src/vsr.zig +19 -5
  74. package/src/tigerbeetle/src/benchmark_array_search.zig +0 -317
  75. package/src/tigerbeetle/src/benchmarks/perf.zig +0 -299
  76. package/src/tigerbeetle/src/vopr_hub/README.md +0 -58
  77. package/src/tigerbeetle/src/vopr_hub/SETUP.md +0 -199
  78. package/src/tigerbeetle/src/vopr_hub/go.mod +0 -3
  79. package/src/tigerbeetle/src/vopr_hub/main.go +0 -1022
  80. package/src/tigerbeetle/src/vopr_hub/scheduler/go.mod +0 -3
  81. package/src/tigerbeetle/src/vopr_hub/scheduler/main.go +0 -403
@@ -1,1022 +0,0 @@
1
- /*
2
- Note: The VOPR Hub must run a VOPR in a separate tigerbeetle directory to prevent it from checking
3
- out a commit that could change the hub itself
4
- In order to parse the output correctly the VOPR must run from within the tigerbeetle directory
5
- To run the VOPR Hub, Zig must be installed and five environmental variables are required:
6
- 1. TIGERBEETLE_DIRECTORY the location of the VOPR's tigerbeetle directory
7
- 2. ISSUE_DIRECTORY where issues are stored on disk
8
- 3. DEVELOPER_TOKEN for access to GitHub
9
- 4. VOPR_HUB_ADDRESS for the IP address to listen on for incoming messages
10
- 5. REPOSITORY_URL to post the GitHub issue
11
- */
12
-
13
- package main
14
-
15
- import (
16
- "bytes"
17
- "crypto/sha256"
18
- "encoding/binary"
19
- "encoding/hex"
20
- "encoding/json"
21
- "errors"
22
- "flag"
23
- "fmt"
24
- "io"
25
- "io/fs"
26
- "net"
27
- "net/http"
28
- "os"
29
- "os/exec"
30
- "regexp"
31
- "strings"
32
- "time"
33
- )
34
-
35
- const max_concurrent_connections = 4
36
- const max_queuing_messages = 100
37
- const length_of_vopr_message = 45
38
- const max_length_of_vopr_output = 8 * 1 << 25
39
-
40
- // GitHub's hard character limit for issues is 65536
41
- const max_github_issue_size = 60000
42
-
43
- const usageFmt = `usage: %s [flags]
44
-
45
- environment:
46
- TIGERBEETLE_DIRECTORY the location of the VOPR's tigerbeetle directory
47
- ISSUE_DIRECTORY where issues are stored on disk
48
- DEVELOPER_TOKEN required for making GitHub API calls
49
- VOPR_HUB_ADDRESS the IP address to listen on for incoming bug reports
50
- REPOSITORY_URL TigerBeetle's GitHub repository URL
51
-
52
- flags:
53
- -debug enabled debug logging
54
- `
55
-
56
- var (
57
- debug_mode bool
58
- tigerbeetle_directory string
59
- issue_directory string
60
- developer_token string
61
- vopr_hub_address string
62
- repository_url string
63
- whitespace_regexpr = regexp.MustCompile(`( ( )+)|(\t)+`)
64
- )
65
-
66
- type Branch struct {
67
- Name string `json:"name"`
68
- }
69
-
70
- type Label struct {
71
- Name string `json:"name"`
72
- }
73
-
74
- type Issue struct {
75
- Labels []Label `json:"labels"`
76
- }
77
-
78
- // An alias for the vopr_message's byte array.
79
- type vopr_message_byte_array [length_of_vopr_message]byte
80
-
81
- type bug_type uint8
82
-
83
- const (
84
- bug_type_correctness = iota + 1
85
- bug_type_liveness
86
- bug_type_crash
87
- )
88
-
89
- type vopr_message struct {
90
- bug bug_type
91
- seed uint64
92
- commit [20]byte
93
- hash [16]byte
94
- }
95
-
96
- // The VOPR's output is stored in a vopr_output struct where certain elements can be extracted and
97
- // processed.
98
- // parameters includes information about the conditions under which TigerBeetle is run.
99
- type vopr_output struct {
100
- logs []byte
101
- stack_trace []byte
102
- stack_trace_hash string
103
- parameters string
104
- seed_passed bool
105
- }
106
-
107
- // Checks if the simulator passed or the bug was confirmed.
108
- func (output *vopr_output) test_passed() bool {
109
- return strings.Contains(string(output.logs), "PASSED")
110
- }
111
-
112
- // The stack trace is moved from output.logs into output.stack_trace.
113
- func (output *vopr_output) extract_stack_trace(message *vopr_message) {
114
- // Only extract stack traces for crash bugs and panic statements for correctness bugs.
115
- if output.seed_passed || message.bug == bug_type_liveness {
116
- return
117
- }
118
- // The stack trace begins on the first line that starts with neither a square bracket nor
119
- // white space.
120
- address_regexpr := regexp.MustCompile(`(\n([^\[\s]))`)
121
- index := address_regexpr.FindIndex(output.logs)
122
- // If an instance is found then the index returns an int array containing the start and end
123
- // index for the match.
124
- if len(index) == 2 {
125
- output.stack_trace = output.logs[index[0]:]
126
- output.logs = output.logs[:index[0]]
127
- log_debug("Stack trace has been extracted", message.hash[:])
128
- } else {
129
- log_debug("No stack trace was found", message.hash[:])
130
- }
131
- }
132
-
133
- // The VOPR's parameters are moved from output.logs into output.parameters.
134
- func (output *vopr_output) extract_parameters(message *vopr_message) {
135
- state_regexpr := regexp.MustCompile(`\[info\] \(state_checker\):[^\[]+`)
136
- index := state_regexpr.FindIndex(output.logs)
137
- // If an instance is found then the index returns an int array containing the start and end
138
- // index for the match.
139
- if len(index) == 2 {
140
- output.parameters = string(output.logs[index[0]:index[1]])
141
- output.parameters = strings.TrimSpace(output.parameters)
142
- output.logs = append(output.logs[:index[0]], output.logs[index[1]:]...)
143
- log_debug("The VOPR's parameters have been extracted", message.hash[:])
144
- } else {
145
- output.parameters = ""
146
- log_debug("No VOPR parameters were found", message.hash[:])
147
- }
148
- }
149
-
150
- // The stack traced is parsed so that all unique paths and memory addresses are removed.
151
- // If the entire stack trace has been recorded, then it's hash will be deterministic.
152
- func (output *vopr_output) parse_stack_trace(message *vopr_message) {
153
- if len(output.stack_trace) > 0 {
154
- path_regexpr := regexp.MustCompile(`(/.*)+/tigerbeetle/`)
155
- memory_address_regexpr := regexp.MustCompile(`: 0x[0-9a-z]* in`)
156
- thread_regexpr := regexp.MustCompile(`thread [0-9]* panic:`)
157
- line_regexpr := regexp.MustCompile(`line [0-9]*: [0-9]*`)
158
- output.stack_trace = path_regexpr.ReplaceAll(output.stack_trace, []byte(""))
159
- output.stack_trace = thread_regexpr.ReplaceAll(output.stack_trace, []byte("thread panic:"))
160
- output.stack_trace = line_regexpr.ReplaceAll(output.stack_trace, []byte(""))
161
- output.stack_trace = memory_address_regexpr.ReplaceAll(output.stack_trace, []byte(": in"))
162
- log_debug("The stack trace has been parsed", message.hash[:])
163
- } else {
164
- log_debug("No stack trace was found to parse", message.hash[:])
165
- }
166
- }
167
-
168
- // Assigns a SHA256 checksum, or a null string, to output.stack_trace_hash.
169
- func (output *vopr_output) hash_stack_trace(message *vopr_message) {
170
- // Check if there is a stack trace
171
- if len(output.stack_trace) > 0 {
172
- check_sum := sha256.Sum256(output.stack_trace)
173
- output.stack_trace_hash = fmt.Sprintf("%x", check_sum)
174
- log_debug("The stack trace has been hashed: "+output.stack_trace_hash, message.hash[:])
175
- } else {
176
- output.stack_trace_hash = ""
177
- log_debug("No stack trace was found to hash", message.hash[:])
178
- }
179
- }
180
-
181
- // Creates the string that forms the body of the GitHub issue.
182
- func (output *vopr_output) create_issue_markdown(message vopr_message) string {
183
- branches := get_branch_names(hex.EncodeToString(message.commit[:]), message.hash[:])
184
-
185
- start := time.Now()
186
- run_vopr_ReleaseSafe(message)
187
- t := time.Now()
188
- duration := t.Sub(start).Round(time.Second)
189
-
190
- // Limit set here to avoid writing only a few characters for any particular section.
191
- const min_useful_length = 100
192
-
193
- // Extract the information about the conditions under which the VOPR runs TigerBeetle.
194
- output.extract_parameters(&message)
195
-
196
- // Remove white space.
197
- output.stack_trace = whitespace_regexpr.ReplaceAll(output.stack_trace, []byte(""))
198
- output.logs = whitespace_regexpr.ReplaceAll(output.logs, []byte(""))
199
- output.parameters = strings.ReplaceAll(output.parameters, " ", "")
200
-
201
- length_of_stack_trace := len(output.stack_trace)
202
- length_of_parameters := len(output.parameters)
203
- length_of_logs := len(output.logs)
204
- remaining_space := max_github_issue_size
205
-
206
- stack_trace := ""
207
- parameters := ""
208
- debug_logs := ""
209
-
210
- // Set the stack trace string.
211
- if length_of_stack_trace > 0 {
212
- if length_of_stack_trace <= max_github_issue_size {
213
- stack_trace = make_markdown_compatible(
214
- "\n```" + string(output.stack_trace[:]) + "\n```\n",
215
- )
216
- remaining_space -= length_of_stack_trace
217
- } else {
218
- // If the stack trace is too large then just capture the beginning of it.
219
- stack_trace = make_markdown_compatible(
220
- "\n```" + string(output.stack_trace[:max_github_issue_size-4]) + "\n```\n",
221
- )
222
- stack_trace += "..."
223
- remaining_space = 0
224
- }
225
- }
226
-
227
- // Set the parameters string.
228
- if length_of_parameters > 0 && remaining_space >= min_useful_length {
229
- if length_of_parameters <= remaining_space {
230
- parameters = make_markdown_compatible(
231
- "\n```" + output.parameters[:] + "\n```\n",
232
- )
233
- remaining_space -= length_of_parameters
234
- } else {
235
- // If the parameters section is too large then just capture the beginning of it.
236
- parameters = make_markdown_compatible(
237
- "\n```"+output.parameters[:remaining_space-4]+"\n```\n",
238
- ) + "..."
239
- remaining_space = 0
240
- }
241
- }
242
-
243
- // Set the debug logs string.
244
- if remaining_space >= min_useful_length {
245
- if length_of_logs < remaining_space {
246
- debug_logs = make_markdown_compatible(
247
- "\n```" + string(output.logs[:]) + "\n```\n",
248
- )
249
- } else {
250
- // Get the tail of the logs.
251
- tail_start_index := length_of_logs - remaining_space
252
- debug_logs = make_markdown_compatible(
253
- "\n```" + string(output.logs[tail_start_index:]) + "\n```\n",
254
- )
255
- }
256
- }
257
-
258
- var body string
259
- if branches != "" {
260
- body += fmt.Sprintf(
261
- "<strong>Branches:</strong> %s<br><br>",
262
- branches,
263
- )
264
- } else {
265
- body += "<strong>Branches:</strong> (unknown branch)<br><br>"
266
- }
267
- body += fmt.Sprintf(
268
- "<strong>Duration to run seed in ReleaseSafe mode:</strong> %s<br><br>",
269
- duration,
270
- )
271
- if len(parameters) > 0 {
272
- body += fmt.Sprintf(
273
- "<strong>Parameters:</strong><br>%s<br>",
274
- parameters,
275
- )
276
- }
277
- if len(stack_trace) > 0 {
278
- body += fmt.Sprintf(
279
- "<strong>Stack Trace:</strong><br>%s<br>",
280
- stack_trace,
281
- )
282
- }
283
- if len(debug_logs) > 0 {
284
- body += fmt.Sprintf(
285
- "<strong>Debug Logs:</strong><br><details><summary>Tail</summary><br>%s</details>",
286
- debug_logs,
287
- )
288
- }
289
-
290
- markdown := fmt.Sprintf(
291
- "<strong>Commit:</strong> ```%s``` <br><br>%s",
292
- hex.EncodeToString(message.commit[:]),
293
- body,
294
- )
295
- return markdown
296
- }
297
-
298
- // The limited buffer ensures that only a specified number of bytes can be written to the buffer.
299
- type size_limited_buffer struct {
300
- byte_budget int
301
- // The buffer being wrapped
302
- buffer *bytes.Buffer
303
- // This channel is used to keep track of when the VOPR reaches its maximum allowed output.
304
- size_reached chan bool
305
- }
306
-
307
- func usage() {
308
- fmt.Fprintf(os.Stderr, usageFmt, os.Args[0])
309
- os.Exit(1)
310
- }
311
-
312
- // The limited buffer honours the Write function in such a way that only a specified number of
313
- // bytes can be written to the buffer. If the maximum number of bytes has been written then no
314
- // more will be added and a value of true is passed to the size_reached channel which acts as a
315
- // signal to kill the process that is doing the writing.
316
- func (limited_buffer *size_limited_buffer) Write(byte_array []byte) (int, error) {
317
- space_left := limited_buffer.byte_budget
318
- limited_buffer.byte_budget -= len(byte_array)
319
- if limited_buffer.byte_budget <= 0 {
320
- if space_left > 0 {
321
- bytes_written, err := limited_buffer.buffer.Write(byte_array[0 : space_left-1])
322
- limited_buffer.size_reached <- true
323
- return bytes_written, err
324
- } else {
325
- limited_buffer.size_reached <- true
326
- return 0, nil
327
- }
328
- } else if space_left > 0 {
329
- return limited_buffer.buffer.Write(byte_array)
330
- } else {
331
- return 0, nil
332
- }
333
- }
334
-
335
- func get_branch_names(commit_sha string, message_hash []byte) string {
336
- branches := []Branch{}
337
- var branch_names string
338
-
339
- get_request, err := http.NewRequest(
340
- "GET",
341
- fmt.Sprintf("%s/commits/%s/branches-where-head", repository_url, commit_sha),
342
- nil,
343
- )
344
- if err != nil {
345
- log_error("unable to create get request to get branch names", message_hash)
346
- panic(err.Error())
347
- }
348
- get_request.Header.Set("Authorization", "token "+developer_token)
349
- res, err := http.DefaultClient.Do(get_request)
350
- if err != nil {
351
- log_error("Failed to send the HTTP request for the GitHub API", message_hash)
352
- panic(err.Error())
353
- }
354
- defer res.Body.Close()
355
-
356
- body, err := io.ReadAll(res.Body)
357
-
358
- if res.StatusCode > 299 {
359
- log_error(
360
- fmt.Sprintf(
361
- "Response failed with status code: %d and\nbody: %s\n",
362
- res.StatusCode,
363
- body,
364
- ),
365
- message_hash,
366
- )
367
- // If there is an error instead of panicking simply return a null string to indicate that
368
- // no branch was found.
369
- // Getting a 422 error when the commit sha doesn't match any head commits.
370
- return ""
371
- }
372
- if err != nil {
373
- log_error("unable to receive a response from GitHub", message_hash)
374
- panic(err.Error())
375
- }
376
-
377
- err = json.Unmarshal(body, &branches)
378
- if err != nil {
379
- log_error("unable to unmarshall json", message_hash)
380
- panic(err.Error())
381
- }
382
-
383
- if len(branches) > 0 {
384
- for _, value := range branches {
385
- branch_names += value.Name + ", "
386
- }
387
- return strings.TrimSuffix(branch_names, ", ")
388
- }
389
-
390
- return ""
391
- }
392
-
393
- func run_vopr_ReleaseSafe(message vopr_message) {
394
- // Runs in ReleaseSafe mode
395
- log_info("Running the VOPR in ReleaseSafe mode...", message.hash[:])
396
- cmd := exec.Command(
397
- "zig/zig",
398
- "build",
399
- "vopr",
400
- "--",
401
- fmt.Sprintf("--seed=%d", message.seed),
402
- "--build-mode=ReleaseSafe",
403
- )
404
- cmd.Dir = tigerbeetle_directory
405
- err := cmd.Run()
406
- if err != nil {
407
- log_error("Failed to run the VOPR in ReleaseSafe mode", message.hash[:])
408
- panic(err.Error())
409
- }
410
- }
411
-
412
- func get_bug_name(bug bug_type) string {
413
- var bug_string string
414
- switch bug {
415
- case bug_type_correctness:
416
- bug_string = "correctness"
417
- case bug_type_liveness:
418
- bug_string = "liveness"
419
- case bug_type_crash:
420
- bug_string = "crash"
421
- default:
422
- panic("unreachable")
423
- }
424
-
425
- return bug_string
426
- }
427
-
428
- // Reads incoming messages (ensuring that only the correct number of bytes are read), decodes and
429
- // validates them before adding the messages to the vopr_message_channel.
430
- func handle_connection(
431
- track_connections chan bool,
432
- connection net.Conn,
433
- vopr_message_channel chan vopr_message,
434
- ) {
435
- // When this function completes, close the connection and decrement the connections count.
436
- defer func() {
437
- connection.Close()
438
- <-track_connections
439
- log_debug("Connection closed", nil)
440
- }()
441
-
442
- var input vopr_message_byte_array
443
-
444
- // If too few bytes were sent then the connection's read deadline will timeout.
445
- total_bytes_read := 0
446
- for total_bytes_read < length_of_vopr_message {
447
- bytes_read, error := connection.Read(input[total_bytes_read:])
448
- if error != nil {
449
- error_message := fmt.Sprintf("Client closed unexpectedly: %s", error.Error())
450
- log_error(error_message, nil)
451
- return
452
- }
453
- total_bytes_read += bytes_read
454
- }
455
-
456
- if total_bytes_read != length_of_vopr_message {
457
- log_error("The input was longer or shorter than expected", nil)
458
- return
459
- }
460
-
461
- // Decodes the byte array into a vopr_message
462
- message, decoding_error := decode_message(input)
463
-
464
- if decoding_error == nil {
465
- log_message := fmt.Sprintf(
466
- "bug: %d commit: %x seed: %d",
467
- message.bug,
468
- message.commit,
469
- message.seed,
470
- )
471
- log_info(log_message, message.hash[:])
472
-
473
- // Checks if there is capacity to process the message.
474
- select {
475
- case vopr_message_channel <- message:
476
- // Reply to client only reply if everything is as expected.
477
- _, error := connection.Write([]byte("1"))
478
- if error != nil {
479
- error_message := fmt.Sprintf("Unable to reply to client: %s", error.Error())
480
- log_error(error_message, message.hash[:])
481
- return
482
- }
483
- default:
484
- log_info("Too many messages already queued, dropping message", message.hash[:])
485
- }
486
- } else {
487
- log_error(decoding_error.Error(), nil)
488
- }
489
- }
490
-
491
- // Decodes the vopr_message_byte_array into a vopr_message struct.
492
- func decode_message(input vopr_message_byte_array) (vopr_message, error) {
493
- var message vopr_message
494
- error := fmt.Errorf("The input received is invalid")
495
-
496
- if len(input) != length_of_vopr_message {
497
- return message, error
498
- }
499
-
500
- // Expect the first 16 bytes of the message to be the first half of a SHA256 hash of the
501
- // remainder of the message.
502
- // If correct, this half of the hash is then also used as a unique identifier of this message
503
- // throughout the logs.
504
- hash := sha256.Sum256(input[16:])
505
- copy(message.hash[:], hash[0:16])
506
- if bytes.Compare(message.hash[:], input[0:16]) != 0 {
507
- checksum_error := fmt.Errorf("Received message with invalid checksum")
508
- return message, checksum_error
509
- }
510
-
511
- // Ensure the bug type is valid.
512
- if input[16] != 1 && input[16] != 2 && input[16] != 3 {
513
- return message, error
514
- }
515
- seed := binary.BigEndian.Uint64(input[17:25])
516
-
517
- // The bug type (1, 2, or 3) is encoded as a uint8.
518
- message.bug = bug_type(input[16])
519
- // The seed is encoded as a uint64.
520
- message.seed = seed
521
- // The GitHub commit hash is remains as a 20 byte array.
522
- copy(message.commit[:], input[25:45])
523
-
524
- return message, nil
525
- }
526
-
527
- // The write functionality is in its own function so it can run in its own Goroutine to allow the
528
- // connection to be closed without delays.
529
- func write_to_vopr_message_channel(message vopr_message, vopr_message_channel chan vopr_message) {
530
- vopr_message_channel <- message
531
- log_debug("Message has been added to channel for processing", message.hash[:])
532
- }
533
-
534
- // Reads and processes messages from the vopr_message_channel channel as they arrive.
535
- // Messages are only sent here after being decoded and validated.
536
- func worker(vopr_message_channel chan vopr_message) {
537
- for message := range vopr_message_channel {
538
- process(message)
539
- log_info("Message processing complete", message.hash[:])
540
- }
541
- }
542
-
543
- // Responsable for running the VOPR, capturing the output, processing it, writing it to file and
544
- // making the GitHub issue.
545
- // It also checks for duplicate issues and ensures the specified commit is available.
546
- func process(message vopr_message) {
547
- // Bugs 1 & 2 don't require a stack trace to be deduped.
548
- if is_duplicate_bug_1_and_2(message) {
549
- return
550
- }
551
-
552
- // Get the number of issues with the `seed` label.
553
- num_issues := get_issues()
554
- // If there are 6 or more VOPR created issues on GitHub then don't continue message processing.
555
- if num_issues >= 6 {
556
- log_info("There are too many open GitHub issues.", message.hash[:])
557
- return
558
- }
559
-
560
- commit_string := hex.EncodeToString(message.commit[:])
561
-
562
- error := checkout_commit(commit_string, message.hash[:])
563
- if error == nil {
564
- log_debug("Successfully checked out commit "+commit_string, message.hash[:])
565
- } else {
566
- error_message := fmt.Sprintf(
567
- "Failed to checkout commit %s: %s", commit_string,
568
- error.Error(),
569
- )
570
- log_error(error_message, message.hash[:])
571
- return
572
- }
573
-
574
- var output vopr_output
575
- run_vopr(message.seed, &output, message.hash[:])
576
- if output.test_passed() {
577
- log_error("The seed unexpectedly passed", message.hash[:])
578
- output.seed_passed = true
579
- } else {
580
- output.seed_passed = false
581
- }
582
-
583
- output.extract_stack_trace(&message)
584
- output.parse_stack_trace(&message)
585
- output.hash_stack_trace(&message)
586
-
587
- issue_file_name := generate_file_name(message, output.stack_trace_hash)
588
-
589
- // Bug 3 requires a stack trace to be deduped
590
- if message.bug == bug_type_crash && is_duplicate(issue_file_name, message.hash[:]) {
591
- return
592
- }
593
-
594
- // Saves report to disk
595
- create_issue_file(issue_file_name, &output, message.hash[:])
596
-
597
- err := create_github_issue(message, &output)
598
- if err != nil {
599
- log_error(
600
- fmt.Sprintf("Failed to create GitHub issue: %s", err.Error()),
601
- message.hash[:],
602
- )
603
- return
604
- }
605
- }
606
-
607
- // Checks if a duplicate issue has already been submitted.
608
- func is_duplicate(issue_file_name string, message_hash []byte) bool {
609
- if _, error := os.Stat(issue_file_name); error == nil {
610
- log_info("Duplicate issue found", message_hash)
611
- return true
612
- } else if errors.Is(error, fs.ErrNotExist) {
613
- return false
614
- } else {
615
- // This is an actual error from the OS/FS. The file may or may not exist, we don't know.
616
- log_error(error.Error(), message_hash)
617
- panic(error.Error())
618
- }
619
- }
620
-
621
- // Bugs 1 and 2 don't require a stack trace for deduping and so they can be deduped before the VOPR
622
- // is run.
623
- func is_duplicate_bug_1_and_2(message vopr_message) bool {
624
- // If bug type 1 or 2 first check if file exists before parsing and hashing the stack trace.
625
- if message.bug == bug_type_correctness || message.bug == bug_type_liveness {
626
- issue_file_name := generate_file_name(message, "")
627
- return is_duplicate(issue_file_name, message.hash[:])
628
- } else {
629
- return false
630
- }
631
- }
632
-
633
- func get_issues() int {
634
- issues := []Issue{}
635
- // All issues created by the VOPR Hub will contain the `seed` label.
636
- get_request, err := http.NewRequest(
637
- "GET",
638
- repository_url+"/issues?labels=seed",
639
- nil,
640
- )
641
- if err != nil {
642
- log_error("unable to create get request to get issues", nil)
643
- panic(err.Error())
644
- }
645
- get_request.Header.Set("Authorization", "token "+developer_token)
646
- res, err := http.DefaultClient.Do(get_request)
647
- if err != nil {
648
- log_error("Failed to send the HTTP request for the GitHub API", nil)
649
- panic(err.Error())
650
- }
651
- defer res.Body.Close()
652
- body, err := io.ReadAll(res.Body)
653
-
654
- if res.StatusCode > 299 {
655
- log_error(fmt.Sprintf(
656
- "Response failed with status code: %d and\nbody: %s\n",
657
- res.StatusCode,
658
- body,
659
- ), nil)
660
- panic(err.Error())
661
- }
662
- if err != nil {
663
- log_error("unable to receive a response from GitHub", nil)
664
- panic(err.Error())
665
- }
666
-
667
- err = json.Unmarshal(body, &issues)
668
- if err != nil {
669
- log_error("unable to unmarshall json", nil)
670
- panic(err.Error())
671
- }
672
- return len(issues)
673
- }
674
-
675
- // Filenames are used to uniquely identify issues for deduping purposes.
676
- // correctness: 1_seed_commithash
677
- // liveness: 2_seed_commithash
678
- // crash: 3_commithash_stacktracehash
679
- func generate_file_name(message vopr_message, stack_trace_hash string) string {
680
- commit_string := hex.EncodeToString(message.commit[:])
681
-
682
- if message.bug == bug_type_correctness || message.bug == bug_type_liveness {
683
- return fmt.Sprintf(
684
- "%s/%d_%d_%s",
685
- issue_directory,
686
- message.bug,
687
- message.seed,
688
- commit_string,
689
- )
690
- } else if message.bug == bug_type_crash {
691
- return fmt.Sprintf(
692
- "%s/%d_%s_%s",
693
- issue_directory,
694
- message.bug,
695
- commit_string,
696
- stack_trace_hash,
697
- )
698
- } else {
699
- panic("unreachable")
700
- }
701
- }
702
-
703
- // Fetch available branches from GitHub and checkout the correct commit if it exists.
704
- func checkout_commit(commit string, message_hash []byte) error {
705
- // Ensures commit is all hexadecimal.
706
- commit_valid, error := regexp.MatchString(`^([0-9a-f]){40}$`, commit)
707
- if error != nil {
708
- panic(error)
709
- } else if !commit_valid {
710
- error = fmt.Errorf("The GitHub commit contained unexpected characters")
711
- log_error(error.Error(), message_hash)
712
- return error
713
- }
714
-
715
- // Git commands need to be run with the TigerBeetle directory as their working_directory.
716
- fetch_command := exec.Command("git", "fetch", "--all")
717
- fetch_command.Dir = tigerbeetle_directory
718
- error = fetch_command.Run()
719
- if error != nil {
720
- error_message := fmt.Sprintf("Failed to run git fetch: %s", error.Error())
721
- log_error(error_message, message_hash)
722
- return error
723
- }
724
-
725
- // Checkout the commit specified in the vopr_message.
726
- checkout_command := exec.Command("git", "checkout", commit)
727
- checkout_command.Dir = tigerbeetle_directory
728
- error = checkout_command.Run()
729
- if error != nil {
730
- error_message := fmt.Sprintf("Failed to run git checkout: %s", error.Error())
731
- log_error(error_message, message_hash)
732
- return error
733
- }
734
-
735
- // Inspect the git logs.
736
- log_command := exec.Command("git", "log", "-1")
737
- log_command.Dir = tigerbeetle_directory
738
- log_output := make([]byte, 47)
739
- log_output, error = log_command.Output()
740
- if error != nil {
741
- error_message := fmt.Sprintf("Failed to run git log: %s", error.Error())
742
- log_error(error_message, message_hash)
743
- return error
744
- }
745
-
746
- // Check the log to determine if the commit has been successfully checked out.
747
- current_commit := string(log_output[0:47])
748
- checkout_successful, error := regexp.MatchString("^commit "+commit, current_commit)
749
- if error != nil {
750
- error_message := fmt.Sprintf(
751
- "Regular expression failure while checking the git logs: %s",
752
- error.Error(),
753
- )
754
- log_error(error_message, message_hash)
755
- return error
756
- }
757
-
758
- if !checkout_successful {
759
- error = fmt.Errorf("Checkout failed")
760
- return error
761
- }
762
-
763
- return nil
764
- }
765
-
766
- // The VOPR is run from the TigerBeetle directory and its output is captured.
767
- func run_vopr(seed uint64, output *vopr_output, message_hash []byte) {
768
- // Create a limited_buffer to read the VOPR output
769
- var vopr_std_err_buffer bytes.Buffer
770
- limited_buffer := size_limited_buffer{
771
- buffer: &vopr_std_err_buffer,
772
- byte_budget: int(max_length_of_vopr_output),
773
- size_reached: make(chan bool),
774
- }
775
-
776
- // The channel monitors if the VOPR completes before the maximum output is reached.
777
- vopr_completed := make(chan error)
778
-
779
- // Runs in Debug mode
780
- log_info("Running the VOPR in Debug mode...", message_hash)
781
- cmd := exec.Command(
782
- "zig/zig",
783
- "build",
784
- "vopr",
785
- "--",
786
- fmt.Sprintf("--seed=%d", seed),
787
- )
788
- cmd.Dir = tigerbeetle_directory
789
- cmd.Stderr = &limited_buffer
790
- // Start() runs asynchronously but is used because it allows a process to be killed.
791
- error := cmd.Start()
792
- if error != nil {
793
- log_error("Failed to start the VOPR in Debug mode", message_hash)
794
- panic(error.Error())
795
- }
796
-
797
- // Wait() runs synchronously. A separate Goroutine is needed to prevent the code from blocking.
798
- // The VOPR might end prematurely instead if its output exceeds the maximum space.
799
- go func() {
800
- result := cmd.Wait()
801
- vopr_completed <- result
802
- }()
803
-
804
- // The code blocks until a value is found in either the vopr_completed or size_reached channel.
805
- select {
806
- case result := <-vopr_completed:
807
- if result != nil {
808
- log_error("The VOPR did not complete successfully", message_hash)
809
- } else {
810
- log_info("The VOPR completed successfully", message_hash)
811
- }
812
- case max_size := <-limited_buffer.size_reached:
813
- if max_size {
814
- cmd.Process.Kill()
815
- }
816
- log_info("The VOPR has completed with a liveness bug", message_hash)
817
- }
818
-
819
- // All results are stored in the output.logs byte array.
820
- output.logs = limited_buffer.buffer.Bytes()
821
- }
822
-
823
- // Writes the debug logs and parsed stack trace to a file on disk.
824
- func create_issue_file(issue_file_name string, output *vopr_output, message_hash []byte) {
825
- full_output := append(output.logs, output.stack_trace...)
826
- error := os.WriteFile(issue_file_name, full_output, 0666)
827
- if error != nil {
828
- error_message := fmt.Sprintf("Failed to write to file: %s", error.Error())
829
- log_error(error_message, message_hash)
830
- panic(error.Error())
831
- } else {
832
- log_message := fmt.Sprintf("Created file: %s", issue_file_name)
833
- log_info(log_message, message_hash)
834
- }
835
- }
836
-
837
- // Submits a GitHub issue that includes the debug logs and parsed stack trace.
838
- func create_github_issue(message vopr_message, output *vopr_output) error {
839
- body := output.create_issue_markdown(message)
840
- if output.seed_passed {
841
- body = "Note this seed passed when it was rerun by the VOPR Hub.<br><br>" + body
842
- }
843
-
844
- bug_string := get_bug_name(message.bug)
845
- title := fmt.Sprintf(
846
- "%s%s: %d",
847
- strings.ToUpper(string(bug_string[0])),
848
- string(bug_string[1:]),
849
- message.seed,
850
- )
851
-
852
- issue_contents, error := json.Marshal(struct {
853
- Title string `json:"title"`
854
- Body string `json:"body"`
855
- Labels []string `json:"labels"`
856
- }{
857
- Title: title,
858
- Body: body,
859
- Labels: []string{"seed", bug_string},
860
- })
861
-
862
- if error != nil {
863
- return error
864
- }
865
-
866
- issue := strings.NewReader(string(issue_contents))
867
- post_request, error := http.NewRequest(
868
- "POST",
869
- repository_url+"/issues",
870
- issue,
871
- )
872
- if error != nil {
873
- log_error("Failed to create the HTTP request for the GitHub API", message.hash[:])
874
- return error
875
- }
876
- post_request.Header.Set("Authorization", "token "+developer_token)
877
-
878
- post_response, error := http.DefaultClient.Do(post_request)
879
- if error != nil {
880
- log_error("Failed to send the HTTP request for the GitHub API", message.hash[:])
881
- return error
882
- }
883
-
884
- defer post_response.Body.Close()
885
-
886
- if post_response.StatusCode < 200 || post_response.StatusCode > 299 {
887
- error_message := fmt.Sprintf(
888
- "Received a non 2xx status code from GitHub. StatusCode: %d. Status: %s",
889
- post_response.StatusCode,
890
- post_response.Status,
891
- )
892
- log_error("Failed to get a reply from the GitHub API", message.hash[:])
893
- return fmt.Errorf(error_message)
894
- }
895
-
896
- log_info(
897
- fmt.Sprintf(
898
- "GitHub issue has been created. Received StatusCode: %d and Status: %s.",
899
- post_response.StatusCode,
900
- post_response.Status,
901
- ),
902
- message.hash[:],
903
- )
904
- return nil
905
- }
906
-
907
- // Escape characters that have special use in Markdown.
908
- func make_markdown_compatible(text string) string {
909
- text = strings.ReplaceAll(text, "^", "")
910
- text = strings.ReplaceAll(text, "\"", "\\\"")
911
- text = strings.ReplaceAll(text, "\t", "")
912
- return text
913
- }
914
-
915
- func load_environment_variable(name string) string {
916
- val, _ := os.LookupEnv(name)
917
- if val == "" {
918
- log_error(fmt.Sprintf("env %s not set", name), nil)
919
- usage()
920
- os.Exit(1)
921
- } else if name == "DEVELOPER_TOKEN" {
922
- log_debug(fmt.Sprintf("env %s=*****", name), nil)
923
- } else {
924
- log_debug(fmt.Sprintf("env %s=%q", name, val), nil)
925
- }
926
- return val
927
- }
928
-
929
- // Ensures that a variable isn't empty or only white space.
930
- func not_empty(variable *string) bool {
931
- *variable = strings.TrimSpace(*variable)
932
- if *variable == "" {
933
- return false
934
- } else {
935
- return true
936
- }
937
- }
938
-
939
- func log_error(message string, vopr_message_hash []byte) {
940
- log_message("error: ", message, vopr_message_hash)
941
- }
942
-
943
- func log_debug(message string, vopr_message_hash []byte) {
944
- if debug_mode {
945
- log_message("debug: ", message, vopr_message_hash)
946
- }
947
- }
948
-
949
- func log_info(message string, vopr_message_hash []byte) {
950
- log_message("info: ", message, vopr_message_hash)
951
- }
952
-
953
- // Formats all the log messages and adds a timestamp to them.
954
- func log_message(log_level string, message string, vopr_message_hash []byte) {
955
- // Gets the current time in UTC.
956
- timestamp := time.Now().UTC().Format(time.RFC3339)
957
- if vopr_message_hash != nil {
958
- // Only use the first 16 bytes of the message hash as the ID.
959
- fmt.Printf("%s %sID:%x %s\n", timestamp, log_level, vopr_message_hash, message)
960
- } else {
961
- fmt.Printf("%s %s%s\n", timestamp, log_level, message)
962
- }
963
- }
964
-
965
- func main() {
966
- // Determine the mode in which to run the VOPR Hub.
967
- flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
968
- flags.BoolVar(&debug_mode, "debug", false, "runs with debugging logs enabled")
969
- if err := flags.Parse(os.Args[1:]); err != nil {
970
- usage()
971
- }
972
-
973
- // Retrieve all the required environment variables up front.
974
- tigerbeetle_directory = load_environment_variable("TIGERBEETLE_DIRECTORY")
975
- issue_directory = load_environment_variable("ISSUE_DIRECTORY")
976
- developer_token = load_environment_variable("DEVELOPER_TOKEN")
977
- vopr_hub_address = load_environment_variable("VOPR_HUB_ADDRESS")
978
- repository_url = load_environment_variable("REPOSITORY_URL")
979
-
980
- // This channel ensures no more than max_concurrent_connections are being handled at one time.
981
- track_connections := make(chan bool, max_concurrent_connections)
982
-
983
- // The channel will receieve fixed-size byte arrays from a VOPR.
984
- vopr_message_channel := make(chan vopr_message, max_queuing_messages)
985
-
986
- // Create a worker Goroutine to call process on each item as it appears in the channel.
987
- go worker(vopr_message_channel)
988
-
989
- listener, error := net.Listen("tcp", vopr_hub_address)
990
- if error != nil {
991
- error_message := fmt.Sprintf("Could not listen on %s: %s", vopr_hub_address, error.Error())
992
- log_error(error_message, nil)
993
- os.Exit(1)
994
- }
995
- defer listener.Close()
996
- log_info("Listening...", nil)
997
-
998
- for {
999
- connection, error := listener.Accept()
1000
- connection.SetReadDeadline(time.Now().Add(10 * time.Second))
1001
- if error != nil {
1002
- error_message := fmt.Sprintf(
1003
- "Unable to set read deadline on the connection: %s",
1004
- error.Error(),
1005
- )
1006
- log_error(error_message, nil)
1007
- connection.Close()
1008
- continue
1009
- }
1010
-
1011
- select {
1012
- case track_connections <- true:
1013
- go handle_connection(track_connections, connection, vopr_message_channel)
1014
- default:
1015
- log_info(
1016
- "Connection closed because there are currently too many open connections",
1017
- nil,
1018
- )
1019
- connection.Close()
1020
- }
1021
- }
1022
- }