tigerbeetle-node 0.11.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.client.node.sha256 +1 -0
- package/package.json +5 -3
- package/src/tigerbeetle/scripts/fuzz_loop.sh +1 -1
- package/src/tigerbeetle/scripts/pre-commit.sh +2 -2
- package/src/tigerbeetle/scripts/validate_docs.sh +17 -0
- package/src/tigerbeetle/src/benchmark.zig +25 -11
- package/src/tigerbeetle/src/c/tb_client/context.zig +248 -47
- package/src/tigerbeetle/src/c/tb_client/echo_client.zig +108 -0
- package/src/tigerbeetle/src/c/tb_client/packet.zig +2 -2
- package/src/tigerbeetle/src/c/tb_client/signal.zig +2 -4
- package/src/tigerbeetle/src/c/tb_client/thread.zig +17 -256
- package/src/tigerbeetle/src/c/tb_client.h +18 -4
- package/src/tigerbeetle/src/c/tb_client.zig +88 -26
- package/src/tigerbeetle/src/c/tb_client_header_test.zig +135 -0
- package/src/tigerbeetle/src/c/test.zig +371 -1
- package/src/tigerbeetle/src/cli.zig +90 -18
- package/src/tigerbeetle/src/config.zig +12 -4
- package/src/tigerbeetle/src/demo.zig +2 -1
- package/src/tigerbeetle/src/demo_01_create_accounts.zig +1 -1
- package/src/tigerbeetle/src/demo_03_create_transfers.zig +13 -0
- package/src/tigerbeetle/src/ewah.zig +11 -33
- package/src/tigerbeetle/src/ewah_benchmark.zig +8 -9
- package/src/tigerbeetle/src/lsm/README.md +97 -3
- package/src/tigerbeetle/src/lsm/compaction.zig +32 -7
- package/src/tigerbeetle/src/{eytzinger_benchmark.zig → lsm/eytzinger_benchmark.zig} +34 -21
- package/src/tigerbeetle/src/lsm/forest_fuzz.zig +34 -32
- package/src/tigerbeetle/src/lsm/grid.zig +39 -21
- package/src/tigerbeetle/src/lsm/groove.zig +1 -0
- package/src/tigerbeetle/src/lsm/k_way_merge.zig +3 -3
- package/src/tigerbeetle/src/lsm/level_iterator.zig +1 -1
- package/src/tigerbeetle/src/lsm/manifest.zig +13 -0
- package/src/tigerbeetle/src/lsm/manifest_level.zig +0 -49
- package/src/tigerbeetle/src/lsm/manifest_log.zig +173 -335
- package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig +665 -0
- package/src/tigerbeetle/src/lsm/node_pool.zig +4 -0
- package/src/tigerbeetle/src/lsm/posted_groove.zig +1 -0
- package/src/tigerbeetle/src/lsm/segmented_array.zig +24 -15
- package/src/tigerbeetle/src/lsm/table.zig +32 -20
- package/src/tigerbeetle/src/lsm/table_immutable.zig +1 -1
- package/src/tigerbeetle/src/lsm/table_iterator.zig +4 -5
- package/src/tigerbeetle/src/lsm/test.zig +13 -2
- package/src/tigerbeetle/src/lsm/tree.zig +45 -7
- package/src/tigerbeetle/src/lsm/tree_fuzz.zig +36 -32
- package/src/tigerbeetle/src/main.zig +69 -13
- package/src/tigerbeetle/src/message_bus.zig +18 -7
- package/src/tigerbeetle/src/message_pool.zig +8 -2
- package/src/tigerbeetle/src/ring_buffer.zig +7 -3
- package/src/tigerbeetle/src/simulator.zig +38 -11
- package/src/tigerbeetle/src/state_machine.zig +48 -23
- package/src/tigerbeetle/src/test/accounting/workload.zig +9 -5
- package/src/tigerbeetle/src/test/cluster.zig +15 -33
- package/src/tigerbeetle/src/test/conductor.zig +2 -1
- package/src/tigerbeetle/src/test/network.zig +45 -19
- package/src/tigerbeetle/src/test/packet_simulator.zig +40 -29
- package/src/tigerbeetle/src/test/state_checker.zig +5 -7
- package/src/tigerbeetle/src/test/storage.zig +453 -110
- package/src/tigerbeetle/src/test/storage_checker.zig +204 -0
- package/src/tigerbeetle/src/tigerbeetle.zig +1 -0
- package/src/tigerbeetle/src/unit_tests.zig +7 -1
- package/src/tigerbeetle/src/util.zig +97 -11
- package/src/tigerbeetle/src/vopr.zig +2 -1
- package/src/tigerbeetle/src/vsr/client.zig +8 -3
- package/src/tigerbeetle/src/vsr/journal.zig +280 -202
- package/src/tigerbeetle/src/vsr/replica.zig +169 -31
- package/src/tigerbeetle/src/vsr/superblock.zig +356 -629
- package/src/tigerbeetle/src/vsr/superblock_client_table.zig +7 -6
- package/src/tigerbeetle/src/vsr/superblock_free_set.zig +414 -151
- package/src/tigerbeetle/src/vsr/superblock_free_set_fuzz.zig +332 -0
- package/src/tigerbeetle/src/vsr/superblock_fuzz.zig +349 -0
- package/src/tigerbeetle/src/vsr/superblock_manifest.zig +44 -9
- package/src/tigerbeetle/src/vsr/superblock_quorums.zig +394 -0
- package/src/tigerbeetle/src/vsr/superblock_quorums_fuzz.zig +312 -0
- package/src/tigerbeetle/src/vsr.zig +19 -5
- package/src/tigerbeetle/src/benchmark_array_search.zig +0 -317
- package/src/tigerbeetle/src/benchmarks/perf.zig +0 -299
- package/src/tigerbeetle/src/vopr_hub/README.md +0 -58
- package/src/tigerbeetle/src/vopr_hub/SETUP.md +0 -199
- package/src/tigerbeetle/src/vopr_hub/go.mod +0 -3
- package/src/tigerbeetle/src/vopr_hub/main.go +0 -1022
- package/src/tigerbeetle/src/vopr_hub/scheduler/go.mod +0 -3
- package/src/tigerbeetle/src/vopr_hub/scheduler/main.go +0 -403
|
@@ -1,1022 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
Note: The VOPR Hub must run a VOPR in a separate tigerbeetle directory to prevent it from checking
|
|
3
|
-
out a commit that could change the hub itself
|
|
4
|
-
In order to parse the output correctly the VOPR must run from within the tigerbeetle directory
|
|
5
|
-
To run the VOPR Hub, Zig must be installed and five environmental variables are required:
|
|
6
|
-
1. TIGERBEETLE_DIRECTORY the location of the VOPR's tigerbeetle directory
|
|
7
|
-
2. ISSUE_DIRECTORY where issues are stored on disk
|
|
8
|
-
3. DEVELOPER_TOKEN for access to GitHub
|
|
9
|
-
4. VOPR_HUB_ADDRESS for the IP address to listen on for incoming messages
|
|
10
|
-
5. REPOSITORY_URL to post the GitHub issue
|
|
11
|
-
*/
|
|
12
|
-
|
|
13
|
-
package main
|
|
14
|
-
|
|
15
|
-
import (
|
|
16
|
-
"bytes"
|
|
17
|
-
"crypto/sha256"
|
|
18
|
-
"encoding/binary"
|
|
19
|
-
"encoding/hex"
|
|
20
|
-
"encoding/json"
|
|
21
|
-
"errors"
|
|
22
|
-
"flag"
|
|
23
|
-
"fmt"
|
|
24
|
-
"io"
|
|
25
|
-
"io/fs"
|
|
26
|
-
"net"
|
|
27
|
-
"net/http"
|
|
28
|
-
"os"
|
|
29
|
-
"os/exec"
|
|
30
|
-
"regexp"
|
|
31
|
-
"strings"
|
|
32
|
-
"time"
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
const max_concurrent_connections = 4
|
|
36
|
-
const max_queuing_messages = 100
|
|
37
|
-
const length_of_vopr_message = 45
|
|
38
|
-
const max_length_of_vopr_output = 8 * 1 << 25
|
|
39
|
-
|
|
40
|
-
// GitHub's hard character limit for issues is 65536
|
|
41
|
-
const max_github_issue_size = 60000
|
|
42
|
-
|
|
43
|
-
const usageFmt = `usage: %s [flags]
|
|
44
|
-
|
|
45
|
-
environment:
|
|
46
|
-
TIGERBEETLE_DIRECTORY the location of the VOPR's tigerbeetle directory
|
|
47
|
-
ISSUE_DIRECTORY where issues are stored on disk
|
|
48
|
-
DEVELOPER_TOKEN required for making GitHub API calls
|
|
49
|
-
VOPR_HUB_ADDRESS the IP address to listen on for incoming bug reports
|
|
50
|
-
REPOSITORY_URL TigerBeetle's GitHub repository URL
|
|
51
|
-
|
|
52
|
-
flags:
|
|
53
|
-
-debug enabled debug logging
|
|
54
|
-
`
|
|
55
|
-
|
|
56
|
-
var (
|
|
57
|
-
debug_mode bool
|
|
58
|
-
tigerbeetle_directory string
|
|
59
|
-
issue_directory string
|
|
60
|
-
developer_token string
|
|
61
|
-
vopr_hub_address string
|
|
62
|
-
repository_url string
|
|
63
|
-
whitespace_regexpr = regexp.MustCompile(`( ( )+)|(\t)+`)
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
type Branch struct {
|
|
67
|
-
Name string `json:"name"`
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
type Label struct {
|
|
71
|
-
Name string `json:"name"`
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
type Issue struct {
|
|
75
|
-
Labels []Label `json:"labels"`
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
// An alias for the vopr_message's byte array.
|
|
79
|
-
type vopr_message_byte_array [length_of_vopr_message]byte
|
|
80
|
-
|
|
81
|
-
type bug_type uint8
|
|
82
|
-
|
|
83
|
-
const (
|
|
84
|
-
bug_type_correctness = iota + 1
|
|
85
|
-
bug_type_liveness
|
|
86
|
-
bug_type_crash
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
type vopr_message struct {
|
|
90
|
-
bug bug_type
|
|
91
|
-
seed uint64
|
|
92
|
-
commit [20]byte
|
|
93
|
-
hash [16]byte
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// The VOPR's output is stored in a vopr_output struct where certain elements can be extracted and
|
|
97
|
-
// processed.
|
|
98
|
-
// parameters includes information about the conditions under which TigerBeetle is run.
|
|
99
|
-
type vopr_output struct {
|
|
100
|
-
logs []byte
|
|
101
|
-
stack_trace []byte
|
|
102
|
-
stack_trace_hash string
|
|
103
|
-
parameters string
|
|
104
|
-
seed_passed bool
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
// Checks if the simulator passed or the bug was confirmed.
|
|
108
|
-
func (output *vopr_output) test_passed() bool {
|
|
109
|
-
return strings.Contains(string(output.logs), "PASSED")
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
// The stack trace is moved from output.logs into output.stack_trace.
|
|
113
|
-
func (output *vopr_output) extract_stack_trace(message *vopr_message) {
|
|
114
|
-
// Only extract stack traces for crash bugs and panic statements for correctness bugs.
|
|
115
|
-
if output.seed_passed || message.bug == bug_type_liveness {
|
|
116
|
-
return
|
|
117
|
-
}
|
|
118
|
-
// The stack trace begins on the first line that starts with neither a square bracket nor
|
|
119
|
-
// white space.
|
|
120
|
-
address_regexpr := regexp.MustCompile(`(\n([^\[\s]))`)
|
|
121
|
-
index := address_regexpr.FindIndex(output.logs)
|
|
122
|
-
// If an instance is found then the index returns an int array containing the start and end
|
|
123
|
-
// index for the match.
|
|
124
|
-
if len(index) == 2 {
|
|
125
|
-
output.stack_trace = output.logs[index[0]:]
|
|
126
|
-
output.logs = output.logs[:index[0]]
|
|
127
|
-
log_debug("Stack trace has been extracted", message.hash[:])
|
|
128
|
-
} else {
|
|
129
|
-
log_debug("No stack trace was found", message.hash[:])
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// The VOPR's parameters are moved from output.logs into output.parameters.
|
|
134
|
-
func (output *vopr_output) extract_parameters(message *vopr_message) {
|
|
135
|
-
state_regexpr := regexp.MustCompile(`\[info\] \(state_checker\):[^\[]+`)
|
|
136
|
-
index := state_regexpr.FindIndex(output.logs)
|
|
137
|
-
// If an instance is found then the index returns an int array containing the start and end
|
|
138
|
-
// index for the match.
|
|
139
|
-
if len(index) == 2 {
|
|
140
|
-
output.parameters = string(output.logs[index[0]:index[1]])
|
|
141
|
-
output.parameters = strings.TrimSpace(output.parameters)
|
|
142
|
-
output.logs = append(output.logs[:index[0]], output.logs[index[1]:]...)
|
|
143
|
-
log_debug("The VOPR's parameters have been extracted", message.hash[:])
|
|
144
|
-
} else {
|
|
145
|
-
output.parameters = ""
|
|
146
|
-
log_debug("No VOPR parameters were found", message.hash[:])
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// The stack traced is parsed so that all unique paths and memory addresses are removed.
|
|
151
|
-
// If the entire stack trace has been recorded, then it's hash will be deterministic.
|
|
152
|
-
func (output *vopr_output) parse_stack_trace(message *vopr_message) {
|
|
153
|
-
if len(output.stack_trace) > 0 {
|
|
154
|
-
path_regexpr := regexp.MustCompile(`(/.*)+/tigerbeetle/`)
|
|
155
|
-
memory_address_regexpr := regexp.MustCompile(`: 0x[0-9a-z]* in`)
|
|
156
|
-
thread_regexpr := regexp.MustCompile(`thread [0-9]* panic:`)
|
|
157
|
-
line_regexpr := regexp.MustCompile(`line [0-9]*: [0-9]*`)
|
|
158
|
-
output.stack_trace = path_regexpr.ReplaceAll(output.stack_trace, []byte(""))
|
|
159
|
-
output.stack_trace = thread_regexpr.ReplaceAll(output.stack_trace, []byte("thread panic:"))
|
|
160
|
-
output.stack_trace = line_regexpr.ReplaceAll(output.stack_trace, []byte(""))
|
|
161
|
-
output.stack_trace = memory_address_regexpr.ReplaceAll(output.stack_trace, []byte(": in"))
|
|
162
|
-
log_debug("The stack trace has been parsed", message.hash[:])
|
|
163
|
-
} else {
|
|
164
|
-
log_debug("No stack trace was found to parse", message.hash[:])
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
// Assigns a SHA256 checksum, or a null string, to output.stack_trace_hash.
|
|
169
|
-
func (output *vopr_output) hash_stack_trace(message *vopr_message) {
|
|
170
|
-
// Check if there is a stack trace
|
|
171
|
-
if len(output.stack_trace) > 0 {
|
|
172
|
-
check_sum := sha256.Sum256(output.stack_trace)
|
|
173
|
-
output.stack_trace_hash = fmt.Sprintf("%x", check_sum)
|
|
174
|
-
log_debug("The stack trace has been hashed: "+output.stack_trace_hash, message.hash[:])
|
|
175
|
-
} else {
|
|
176
|
-
output.stack_trace_hash = ""
|
|
177
|
-
log_debug("No stack trace was found to hash", message.hash[:])
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// Creates the string that forms the body of the GitHub issue.
|
|
182
|
-
func (output *vopr_output) create_issue_markdown(message vopr_message) string {
|
|
183
|
-
branches := get_branch_names(hex.EncodeToString(message.commit[:]), message.hash[:])
|
|
184
|
-
|
|
185
|
-
start := time.Now()
|
|
186
|
-
run_vopr_ReleaseSafe(message)
|
|
187
|
-
t := time.Now()
|
|
188
|
-
duration := t.Sub(start).Round(time.Second)
|
|
189
|
-
|
|
190
|
-
// Limit set here to avoid writing only a few characters for any particular section.
|
|
191
|
-
const min_useful_length = 100
|
|
192
|
-
|
|
193
|
-
// Extract the information about the conditions under which the VOPR runs TigerBeetle.
|
|
194
|
-
output.extract_parameters(&message)
|
|
195
|
-
|
|
196
|
-
// Remove white space.
|
|
197
|
-
output.stack_trace = whitespace_regexpr.ReplaceAll(output.stack_trace, []byte(""))
|
|
198
|
-
output.logs = whitespace_regexpr.ReplaceAll(output.logs, []byte(""))
|
|
199
|
-
output.parameters = strings.ReplaceAll(output.parameters, " ", "")
|
|
200
|
-
|
|
201
|
-
length_of_stack_trace := len(output.stack_trace)
|
|
202
|
-
length_of_parameters := len(output.parameters)
|
|
203
|
-
length_of_logs := len(output.logs)
|
|
204
|
-
remaining_space := max_github_issue_size
|
|
205
|
-
|
|
206
|
-
stack_trace := ""
|
|
207
|
-
parameters := ""
|
|
208
|
-
debug_logs := ""
|
|
209
|
-
|
|
210
|
-
// Set the stack trace string.
|
|
211
|
-
if length_of_stack_trace > 0 {
|
|
212
|
-
if length_of_stack_trace <= max_github_issue_size {
|
|
213
|
-
stack_trace = make_markdown_compatible(
|
|
214
|
-
"\n```" + string(output.stack_trace[:]) + "\n```\n",
|
|
215
|
-
)
|
|
216
|
-
remaining_space -= length_of_stack_trace
|
|
217
|
-
} else {
|
|
218
|
-
// If the stack trace is too large then just capture the beginning of it.
|
|
219
|
-
stack_trace = make_markdown_compatible(
|
|
220
|
-
"\n```" + string(output.stack_trace[:max_github_issue_size-4]) + "\n```\n",
|
|
221
|
-
)
|
|
222
|
-
stack_trace += "..."
|
|
223
|
-
remaining_space = 0
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// Set the parameters string.
|
|
228
|
-
if length_of_parameters > 0 && remaining_space >= min_useful_length {
|
|
229
|
-
if length_of_parameters <= remaining_space {
|
|
230
|
-
parameters = make_markdown_compatible(
|
|
231
|
-
"\n```" + output.parameters[:] + "\n```\n",
|
|
232
|
-
)
|
|
233
|
-
remaining_space -= length_of_parameters
|
|
234
|
-
} else {
|
|
235
|
-
// If the parameters section is too large then just capture the beginning of it.
|
|
236
|
-
parameters = make_markdown_compatible(
|
|
237
|
-
"\n```"+output.parameters[:remaining_space-4]+"\n```\n",
|
|
238
|
-
) + "..."
|
|
239
|
-
remaining_space = 0
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
// Set the debug logs string.
|
|
244
|
-
if remaining_space >= min_useful_length {
|
|
245
|
-
if length_of_logs < remaining_space {
|
|
246
|
-
debug_logs = make_markdown_compatible(
|
|
247
|
-
"\n```" + string(output.logs[:]) + "\n```\n",
|
|
248
|
-
)
|
|
249
|
-
} else {
|
|
250
|
-
// Get the tail of the logs.
|
|
251
|
-
tail_start_index := length_of_logs - remaining_space
|
|
252
|
-
debug_logs = make_markdown_compatible(
|
|
253
|
-
"\n```" + string(output.logs[tail_start_index:]) + "\n```\n",
|
|
254
|
-
)
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
var body string
|
|
259
|
-
if branches != "" {
|
|
260
|
-
body += fmt.Sprintf(
|
|
261
|
-
"<strong>Branches:</strong> %s<br><br>",
|
|
262
|
-
branches,
|
|
263
|
-
)
|
|
264
|
-
} else {
|
|
265
|
-
body += "<strong>Branches:</strong> (unknown branch)<br><br>"
|
|
266
|
-
}
|
|
267
|
-
body += fmt.Sprintf(
|
|
268
|
-
"<strong>Duration to run seed in ReleaseSafe mode:</strong> %s<br><br>",
|
|
269
|
-
duration,
|
|
270
|
-
)
|
|
271
|
-
if len(parameters) > 0 {
|
|
272
|
-
body += fmt.Sprintf(
|
|
273
|
-
"<strong>Parameters:</strong><br>%s<br>",
|
|
274
|
-
parameters,
|
|
275
|
-
)
|
|
276
|
-
}
|
|
277
|
-
if len(stack_trace) > 0 {
|
|
278
|
-
body += fmt.Sprintf(
|
|
279
|
-
"<strong>Stack Trace:</strong><br>%s<br>",
|
|
280
|
-
stack_trace,
|
|
281
|
-
)
|
|
282
|
-
}
|
|
283
|
-
if len(debug_logs) > 0 {
|
|
284
|
-
body += fmt.Sprintf(
|
|
285
|
-
"<strong>Debug Logs:</strong><br><details><summary>Tail</summary><br>%s</details>",
|
|
286
|
-
debug_logs,
|
|
287
|
-
)
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
markdown := fmt.Sprintf(
|
|
291
|
-
"<strong>Commit:</strong> ```%s``` <br><br>%s",
|
|
292
|
-
hex.EncodeToString(message.commit[:]),
|
|
293
|
-
body,
|
|
294
|
-
)
|
|
295
|
-
return markdown
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
// The limited buffer ensures that only a specified number of bytes can be written to the buffer.
|
|
299
|
-
type size_limited_buffer struct {
|
|
300
|
-
byte_budget int
|
|
301
|
-
// The buffer being wrapped
|
|
302
|
-
buffer *bytes.Buffer
|
|
303
|
-
// This channel is used to keep track of when the VOPR reaches its maximum allowed output.
|
|
304
|
-
size_reached chan bool
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
func usage() {
|
|
308
|
-
fmt.Fprintf(os.Stderr, usageFmt, os.Args[0])
|
|
309
|
-
os.Exit(1)
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
// The limited buffer honours the Write function in such a way that only a specified number of
|
|
313
|
-
// bytes can be written to the buffer. If the maximum number of bytes has been written then no
|
|
314
|
-
// more will be added and a value of true is passed to the size_reached channel which acts as a
|
|
315
|
-
// signal to kill the process that is doing the writing.
|
|
316
|
-
func (limited_buffer *size_limited_buffer) Write(byte_array []byte) (int, error) {
|
|
317
|
-
space_left := limited_buffer.byte_budget
|
|
318
|
-
limited_buffer.byte_budget -= len(byte_array)
|
|
319
|
-
if limited_buffer.byte_budget <= 0 {
|
|
320
|
-
if space_left > 0 {
|
|
321
|
-
bytes_written, err := limited_buffer.buffer.Write(byte_array[0 : space_left-1])
|
|
322
|
-
limited_buffer.size_reached <- true
|
|
323
|
-
return bytes_written, err
|
|
324
|
-
} else {
|
|
325
|
-
limited_buffer.size_reached <- true
|
|
326
|
-
return 0, nil
|
|
327
|
-
}
|
|
328
|
-
} else if space_left > 0 {
|
|
329
|
-
return limited_buffer.buffer.Write(byte_array)
|
|
330
|
-
} else {
|
|
331
|
-
return 0, nil
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
func get_branch_names(commit_sha string, message_hash []byte) string {
|
|
336
|
-
branches := []Branch{}
|
|
337
|
-
var branch_names string
|
|
338
|
-
|
|
339
|
-
get_request, err := http.NewRequest(
|
|
340
|
-
"GET",
|
|
341
|
-
fmt.Sprintf("%s/commits/%s/branches-where-head", repository_url, commit_sha),
|
|
342
|
-
nil,
|
|
343
|
-
)
|
|
344
|
-
if err != nil {
|
|
345
|
-
log_error("unable to create get request to get branch names", message_hash)
|
|
346
|
-
panic(err.Error())
|
|
347
|
-
}
|
|
348
|
-
get_request.Header.Set("Authorization", "token "+developer_token)
|
|
349
|
-
res, err := http.DefaultClient.Do(get_request)
|
|
350
|
-
if err != nil {
|
|
351
|
-
log_error("Failed to send the HTTP request for the GitHub API", message_hash)
|
|
352
|
-
panic(err.Error())
|
|
353
|
-
}
|
|
354
|
-
defer res.Body.Close()
|
|
355
|
-
|
|
356
|
-
body, err := io.ReadAll(res.Body)
|
|
357
|
-
|
|
358
|
-
if res.StatusCode > 299 {
|
|
359
|
-
log_error(
|
|
360
|
-
fmt.Sprintf(
|
|
361
|
-
"Response failed with status code: %d and\nbody: %s\n",
|
|
362
|
-
res.StatusCode,
|
|
363
|
-
body,
|
|
364
|
-
),
|
|
365
|
-
message_hash,
|
|
366
|
-
)
|
|
367
|
-
// If there is an error instead of panicking simply return a null string to indicate that
|
|
368
|
-
// no branch was found.
|
|
369
|
-
// Getting a 422 error when the commit sha doesn't match any head commits.
|
|
370
|
-
return ""
|
|
371
|
-
}
|
|
372
|
-
if err != nil {
|
|
373
|
-
log_error("unable to receive a response from GitHub", message_hash)
|
|
374
|
-
panic(err.Error())
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
err = json.Unmarshal(body, &branches)
|
|
378
|
-
if err != nil {
|
|
379
|
-
log_error("unable to unmarshall json", message_hash)
|
|
380
|
-
panic(err.Error())
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
if len(branches) > 0 {
|
|
384
|
-
for _, value := range branches {
|
|
385
|
-
branch_names += value.Name + ", "
|
|
386
|
-
}
|
|
387
|
-
return strings.TrimSuffix(branch_names, ", ")
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
return ""
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
func run_vopr_ReleaseSafe(message vopr_message) {
|
|
394
|
-
// Runs in ReleaseSafe mode
|
|
395
|
-
log_info("Running the VOPR in ReleaseSafe mode...", message.hash[:])
|
|
396
|
-
cmd := exec.Command(
|
|
397
|
-
"zig/zig",
|
|
398
|
-
"build",
|
|
399
|
-
"vopr",
|
|
400
|
-
"--",
|
|
401
|
-
fmt.Sprintf("--seed=%d", message.seed),
|
|
402
|
-
"--build-mode=ReleaseSafe",
|
|
403
|
-
)
|
|
404
|
-
cmd.Dir = tigerbeetle_directory
|
|
405
|
-
err := cmd.Run()
|
|
406
|
-
if err != nil {
|
|
407
|
-
log_error("Failed to run the VOPR in ReleaseSafe mode", message.hash[:])
|
|
408
|
-
panic(err.Error())
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
func get_bug_name(bug bug_type) string {
|
|
413
|
-
var bug_string string
|
|
414
|
-
switch bug {
|
|
415
|
-
case bug_type_correctness:
|
|
416
|
-
bug_string = "correctness"
|
|
417
|
-
case bug_type_liveness:
|
|
418
|
-
bug_string = "liveness"
|
|
419
|
-
case bug_type_crash:
|
|
420
|
-
bug_string = "crash"
|
|
421
|
-
default:
|
|
422
|
-
panic("unreachable")
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
return bug_string
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
// Reads incoming messages (ensuring that only the correct number of bytes are read), decodes and
|
|
429
|
-
// validates them before adding the messages to the vopr_message_channel.
|
|
430
|
-
func handle_connection(
|
|
431
|
-
track_connections chan bool,
|
|
432
|
-
connection net.Conn,
|
|
433
|
-
vopr_message_channel chan vopr_message,
|
|
434
|
-
) {
|
|
435
|
-
// When this function completes, close the connection and decrement the connections count.
|
|
436
|
-
defer func() {
|
|
437
|
-
connection.Close()
|
|
438
|
-
<-track_connections
|
|
439
|
-
log_debug("Connection closed", nil)
|
|
440
|
-
}()
|
|
441
|
-
|
|
442
|
-
var input vopr_message_byte_array
|
|
443
|
-
|
|
444
|
-
// If too few bytes were sent then the connection's read deadline will timeout.
|
|
445
|
-
total_bytes_read := 0
|
|
446
|
-
for total_bytes_read < length_of_vopr_message {
|
|
447
|
-
bytes_read, error := connection.Read(input[total_bytes_read:])
|
|
448
|
-
if error != nil {
|
|
449
|
-
error_message := fmt.Sprintf("Client closed unexpectedly: %s", error.Error())
|
|
450
|
-
log_error(error_message, nil)
|
|
451
|
-
return
|
|
452
|
-
}
|
|
453
|
-
total_bytes_read += bytes_read
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
if total_bytes_read != length_of_vopr_message {
|
|
457
|
-
log_error("The input was longer or shorter than expected", nil)
|
|
458
|
-
return
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
// Decodes the byte array into a vopr_message
|
|
462
|
-
message, decoding_error := decode_message(input)
|
|
463
|
-
|
|
464
|
-
if decoding_error == nil {
|
|
465
|
-
log_message := fmt.Sprintf(
|
|
466
|
-
"bug: %d commit: %x seed: %d",
|
|
467
|
-
message.bug,
|
|
468
|
-
message.commit,
|
|
469
|
-
message.seed,
|
|
470
|
-
)
|
|
471
|
-
log_info(log_message, message.hash[:])
|
|
472
|
-
|
|
473
|
-
// Checks if there is capacity to process the message.
|
|
474
|
-
select {
|
|
475
|
-
case vopr_message_channel <- message:
|
|
476
|
-
// Reply to client only reply if everything is as expected.
|
|
477
|
-
_, error := connection.Write([]byte("1"))
|
|
478
|
-
if error != nil {
|
|
479
|
-
error_message := fmt.Sprintf("Unable to reply to client: %s", error.Error())
|
|
480
|
-
log_error(error_message, message.hash[:])
|
|
481
|
-
return
|
|
482
|
-
}
|
|
483
|
-
default:
|
|
484
|
-
log_info("Too many messages already queued, dropping message", message.hash[:])
|
|
485
|
-
}
|
|
486
|
-
} else {
|
|
487
|
-
log_error(decoding_error.Error(), nil)
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
// Decodes the vopr_message_byte_array into a vopr_message struct.
|
|
492
|
-
func decode_message(input vopr_message_byte_array) (vopr_message, error) {
|
|
493
|
-
var message vopr_message
|
|
494
|
-
error := fmt.Errorf("The input received is invalid")
|
|
495
|
-
|
|
496
|
-
if len(input) != length_of_vopr_message {
|
|
497
|
-
return message, error
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
// Expect the first 16 bytes of the message to be the first half of a SHA256 hash of the
|
|
501
|
-
// remainder of the message.
|
|
502
|
-
// If correct, this half of the hash is then also used as a unique identifier of this message
|
|
503
|
-
// throughout the logs.
|
|
504
|
-
hash := sha256.Sum256(input[16:])
|
|
505
|
-
copy(message.hash[:], hash[0:16])
|
|
506
|
-
if bytes.Compare(message.hash[:], input[0:16]) != 0 {
|
|
507
|
-
checksum_error := fmt.Errorf("Received message with invalid checksum")
|
|
508
|
-
return message, checksum_error
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
// Ensure the bug type is valid.
|
|
512
|
-
if input[16] != 1 && input[16] != 2 && input[16] != 3 {
|
|
513
|
-
return message, error
|
|
514
|
-
}
|
|
515
|
-
seed := binary.BigEndian.Uint64(input[17:25])
|
|
516
|
-
|
|
517
|
-
// The bug type (1, 2, or 3) is encoded as a uint8.
|
|
518
|
-
message.bug = bug_type(input[16])
|
|
519
|
-
// The seed is encoded as a uint64.
|
|
520
|
-
message.seed = seed
|
|
521
|
-
// The GitHub commit hash is remains as a 20 byte array.
|
|
522
|
-
copy(message.commit[:], input[25:45])
|
|
523
|
-
|
|
524
|
-
return message, nil
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
// The write functionality is in its own function so it can run in its own Goroutine to allow the
|
|
528
|
-
// connection to be closed without delays.
|
|
529
|
-
func write_to_vopr_message_channel(message vopr_message, vopr_message_channel chan vopr_message) {
|
|
530
|
-
vopr_message_channel <- message
|
|
531
|
-
log_debug("Message has been added to channel for processing", message.hash[:])
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
// Reads and processes messages from the vopr_message_channel channel as they arrive.
|
|
535
|
-
// Messages are only sent here after being decoded and validated.
|
|
536
|
-
func worker(vopr_message_channel chan vopr_message) {
|
|
537
|
-
for message := range vopr_message_channel {
|
|
538
|
-
process(message)
|
|
539
|
-
log_info("Message processing complete", message.hash[:])
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
// Responsable for running the VOPR, capturing the output, processing it, writing it to file and
|
|
544
|
-
// making the GitHub issue.
|
|
545
|
-
// It also checks for duplicate issues and ensures the specified commit is available.
|
|
546
|
-
func process(message vopr_message) {
|
|
547
|
-
// Bugs 1 & 2 don't require a stack trace to be deduped.
|
|
548
|
-
if is_duplicate_bug_1_and_2(message) {
|
|
549
|
-
return
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
// Get the number of issues with the `seed` label.
|
|
553
|
-
num_issues := get_issues()
|
|
554
|
-
// If there are 6 or more VOPR created issues on GitHub then don't continue message processing.
|
|
555
|
-
if num_issues >= 6 {
|
|
556
|
-
log_info("There are too many open GitHub issues.", message.hash[:])
|
|
557
|
-
return
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
commit_string := hex.EncodeToString(message.commit[:])
|
|
561
|
-
|
|
562
|
-
error := checkout_commit(commit_string, message.hash[:])
|
|
563
|
-
if error == nil {
|
|
564
|
-
log_debug("Successfully checked out commit "+commit_string, message.hash[:])
|
|
565
|
-
} else {
|
|
566
|
-
error_message := fmt.Sprintf(
|
|
567
|
-
"Failed to checkout commit %s: %s", commit_string,
|
|
568
|
-
error.Error(),
|
|
569
|
-
)
|
|
570
|
-
log_error(error_message, message.hash[:])
|
|
571
|
-
return
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
var output vopr_output
|
|
575
|
-
run_vopr(message.seed, &output, message.hash[:])
|
|
576
|
-
if output.test_passed() {
|
|
577
|
-
log_error("The seed unexpectedly passed", message.hash[:])
|
|
578
|
-
output.seed_passed = true
|
|
579
|
-
} else {
|
|
580
|
-
output.seed_passed = false
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
output.extract_stack_trace(&message)
|
|
584
|
-
output.parse_stack_trace(&message)
|
|
585
|
-
output.hash_stack_trace(&message)
|
|
586
|
-
|
|
587
|
-
issue_file_name := generate_file_name(message, output.stack_trace_hash)
|
|
588
|
-
|
|
589
|
-
// Bug 3 requires a stack trace to be deduped
|
|
590
|
-
if message.bug == bug_type_crash && is_duplicate(issue_file_name, message.hash[:]) {
|
|
591
|
-
return
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
// Saves report to disk
|
|
595
|
-
create_issue_file(issue_file_name, &output, message.hash[:])
|
|
596
|
-
|
|
597
|
-
err := create_github_issue(message, &output)
|
|
598
|
-
if err != nil {
|
|
599
|
-
log_error(
|
|
600
|
-
fmt.Sprintf("Failed to create GitHub issue: %s", err.Error()),
|
|
601
|
-
message.hash[:],
|
|
602
|
-
)
|
|
603
|
-
return
|
|
604
|
-
}
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
// Checks if a duplicate issue has already been submitted.
|
|
608
|
-
func is_duplicate(issue_file_name string, message_hash []byte) bool {
|
|
609
|
-
if _, error := os.Stat(issue_file_name); error == nil {
|
|
610
|
-
log_info("Duplicate issue found", message_hash)
|
|
611
|
-
return true
|
|
612
|
-
} else if errors.Is(error, fs.ErrNotExist) {
|
|
613
|
-
return false
|
|
614
|
-
} else {
|
|
615
|
-
// This is an actual error from the OS/FS. The file may or may not exist, we don't know.
|
|
616
|
-
log_error(error.Error(), message_hash)
|
|
617
|
-
panic(error.Error())
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
// Bugs 1 and 2 don't require a stack trace for deduping and so they can be deduped before the VOPR
|
|
622
|
-
// is run.
|
|
623
|
-
func is_duplicate_bug_1_and_2(message vopr_message) bool {
|
|
624
|
-
// If bug type 1 or 2 first check if file exists before parsing and hashing the stack trace.
|
|
625
|
-
if message.bug == bug_type_correctness || message.bug == bug_type_liveness {
|
|
626
|
-
issue_file_name := generate_file_name(message, "")
|
|
627
|
-
return is_duplicate(issue_file_name, message.hash[:])
|
|
628
|
-
} else {
|
|
629
|
-
return false
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
func get_issues() int {
|
|
634
|
-
issues := []Issue{}
|
|
635
|
-
// All issues created by the VOPR Hub will contain the `seed` label.
|
|
636
|
-
get_request, err := http.NewRequest(
|
|
637
|
-
"GET",
|
|
638
|
-
repository_url+"/issues?labels=seed",
|
|
639
|
-
nil,
|
|
640
|
-
)
|
|
641
|
-
if err != nil {
|
|
642
|
-
log_error("unable to create get request to get issues", nil)
|
|
643
|
-
panic(err.Error())
|
|
644
|
-
}
|
|
645
|
-
get_request.Header.Set("Authorization", "token "+developer_token)
|
|
646
|
-
res, err := http.DefaultClient.Do(get_request)
|
|
647
|
-
if err != nil {
|
|
648
|
-
log_error("Failed to send the HTTP request for the GitHub API", nil)
|
|
649
|
-
panic(err.Error())
|
|
650
|
-
}
|
|
651
|
-
defer res.Body.Close()
|
|
652
|
-
body, err := io.ReadAll(res.Body)
|
|
653
|
-
|
|
654
|
-
if res.StatusCode > 299 {
|
|
655
|
-
log_error(fmt.Sprintf(
|
|
656
|
-
"Response failed with status code: %d and\nbody: %s\n",
|
|
657
|
-
res.StatusCode,
|
|
658
|
-
body,
|
|
659
|
-
), nil)
|
|
660
|
-
panic(err.Error())
|
|
661
|
-
}
|
|
662
|
-
if err != nil {
|
|
663
|
-
log_error("unable to receive a response from GitHub", nil)
|
|
664
|
-
panic(err.Error())
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
err = json.Unmarshal(body, &issues)
|
|
668
|
-
if err != nil {
|
|
669
|
-
log_error("unable to unmarshall json", nil)
|
|
670
|
-
panic(err.Error())
|
|
671
|
-
}
|
|
672
|
-
return len(issues)
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
// Filenames are used to uniquely identify issues for deduping purposes.
|
|
676
|
-
// correctness: 1_seed_commithash
|
|
677
|
-
// liveness: 2_seed_commithash
|
|
678
|
-
// crash: 3_commithash_stacktracehash
|
|
679
|
-
func generate_file_name(message vopr_message, stack_trace_hash string) string {
|
|
680
|
-
commit_string := hex.EncodeToString(message.commit[:])
|
|
681
|
-
|
|
682
|
-
if message.bug == bug_type_correctness || message.bug == bug_type_liveness {
|
|
683
|
-
return fmt.Sprintf(
|
|
684
|
-
"%s/%d_%d_%s",
|
|
685
|
-
issue_directory,
|
|
686
|
-
message.bug,
|
|
687
|
-
message.seed,
|
|
688
|
-
commit_string,
|
|
689
|
-
)
|
|
690
|
-
} else if message.bug == bug_type_crash {
|
|
691
|
-
return fmt.Sprintf(
|
|
692
|
-
"%s/%d_%s_%s",
|
|
693
|
-
issue_directory,
|
|
694
|
-
message.bug,
|
|
695
|
-
commit_string,
|
|
696
|
-
stack_trace_hash,
|
|
697
|
-
)
|
|
698
|
-
} else {
|
|
699
|
-
panic("unreachable")
|
|
700
|
-
}
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
// Fetch available branches from GitHub and checkout the correct commit if it exists.
|
|
704
|
-
func checkout_commit(commit string, message_hash []byte) error {
|
|
705
|
-
// Ensures commit is all hexadecimal.
|
|
706
|
-
commit_valid, error := regexp.MatchString(`^([0-9a-f]){40}$`, commit)
|
|
707
|
-
if error != nil {
|
|
708
|
-
panic(error)
|
|
709
|
-
} else if !commit_valid {
|
|
710
|
-
error = fmt.Errorf("The GitHub commit contained unexpected characters")
|
|
711
|
-
log_error(error.Error(), message_hash)
|
|
712
|
-
return error
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
// Git commands need to be run with the TigerBeetle directory as their working_directory.
|
|
716
|
-
fetch_command := exec.Command("git", "fetch", "--all")
|
|
717
|
-
fetch_command.Dir = tigerbeetle_directory
|
|
718
|
-
error = fetch_command.Run()
|
|
719
|
-
if error != nil {
|
|
720
|
-
error_message := fmt.Sprintf("Failed to run git fetch: %s", error.Error())
|
|
721
|
-
log_error(error_message, message_hash)
|
|
722
|
-
return error
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
// Checkout the commit specified in the vopr_message.
|
|
726
|
-
checkout_command := exec.Command("git", "checkout", commit)
|
|
727
|
-
checkout_command.Dir = tigerbeetle_directory
|
|
728
|
-
error = checkout_command.Run()
|
|
729
|
-
if error != nil {
|
|
730
|
-
error_message := fmt.Sprintf("Failed to run git checkout: %s", error.Error())
|
|
731
|
-
log_error(error_message, message_hash)
|
|
732
|
-
return error
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
// Inspect the git logs.
|
|
736
|
-
log_command := exec.Command("git", "log", "-1")
|
|
737
|
-
log_command.Dir = tigerbeetle_directory
|
|
738
|
-
log_output := make([]byte, 47)
|
|
739
|
-
log_output, error = log_command.Output()
|
|
740
|
-
if error != nil {
|
|
741
|
-
error_message := fmt.Sprintf("Failed to run git log: %s", error.Error())
|
|
742
|
-
log_error(error_message, message_hash)
|
|
743
|
-
return error
|
|
744
|
-
}
|
|
745
|
-
|
|
746
|
-
// Check the log to determine if the commit has been successfully checked out.
|
|
747
|
-
current_commit := string(log_output[0:47])
|
|
748
|
-
checkout_successful, error := regexp.MatchString("^commit "+commit, current_commit)
|
|
749
|
-
if error != nil {
|
|
750
|
-
error_message := fmt.Sprintf(
|
|
751
|
-
"Regular expression failure while checking the git logs: %s",
|
|
752
|
-
error.Error(),
|
|
753
|
-
)
|
|
754
|
-
log_error(error_message, message_hash)
|
|
755
|
-
return error
|
|
756
|
-
}
|
|
757
|
-
|
|
758
|
-
if !checkout_successful {
|
|
759
|
-
error = fmt.Errorf("Checkout failed")
|
|
760
|
-
return error
|
|
761
|
-
}
|
|
762
|
-
|
|
763
|
-
return nil
|
|
764
|
-
}
|
|
765
|
-
|
|
766
|
-
// The VOPR is run from the TigerBeetle directory and its output is captured.
|
|
767
|
-
func run_vopr(seed uint64, output *vopr_output, message_hash []byte) {
|
|
768
|
-
// Create a limited_buffer to read the VOPR output
|
|
769
|
-
var vopr_std_err_buffer bytes.Buffer
|
|
770
|
-
limited_buffer := size_limited_buffer{
|
|
771
|
-
buffer: &vopr_std_err_buffer,
|
|
772
|
-
byte_budget: int(max_length_of_vopr_output),
|
|
773
|
-
size_reached: make(chan bool),
|
|
774
|
-
}
|
|
775
|
-
|
|
776
|
-
// The channel monitors if the VOPR completes before the maximum output is reached.
|
|
777
|
-
vopr_completed := make(chan error)
|
|
778
|
-
|
|
779
|
-
// Runs in Debug mode
|
|
780
|
-
log_info("Running the VOPR in Debug mode...", message_hash)
|
|
781
|
-
cmd := exec.Command(
|
|
782
|
-
"zig/zig",
|
|
783
|
-
"build",
|
|
784
|
-
"vopr",
|
|
785
|
-
"--",
|
|
786
|
-
fmt.Sprintf("--seed=%d", seed),
|
|
787
|
-
)
|
|
788
|
-
cmd.Dir = tigerbeetle_directory
|
|
789
|
-
cmd.Stderr = &limited_buffer
|
|
790
|
-
// Start() runs asynchronously but is used because it allows a process to be killed.
|
|
791
|
-
error := cmd.Start()
|
|
792
|
-
if error != nil {
|
|
793
|
-
log_error("Failed to start the VOPR in Debug mode", message_hash)
|
|
794
|
-
panic(error.Error())
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
// Wait() runs synchronously. A separate Goroutine is needed to prevent the code from blocking.
|
|
798
|
-
// The VOPR might end prematurely instead if its output exceeds the maximum space.
|
|
799
|
-
go func() {
|
|
800
|
-
result := cmd.Wait()
|
|
801
|
-
vopr_completed <- result
|
|
802
|
-
}()
|
|
803
|
-
|
|
804
|
-
// The code blocks until a value is found in either the vopr_completed or size_reached channel.
|
|
805
|
-
select {
|
|
806
|
-
case result := <-vopr_completed:
|
|
807
|
-
if result != nil {
|
|
808
|
-
log_error("The VOPR did not complete successfully", message_hash)
|
|
809
|
-
} else {
|
|
810
|
-
log_info("The VOPR completed successfully", message_hash)
|
|
811
|
-
}
|
|
812
|
-
case max_size := <-limited_buffer.size_reached:
|
|
813
|
-
if max_size {
|
|
814
|
-
cmd.Process.Kill()
|
|
815
|
-
}
|
|
816
|
-
log_info("The VOPR has completed with a liveness bug", message_hash)
|
|
817
|
-
}
|
|
818
|
-
|
|
819
|
-
// All results are stored in the output.logs byte array.
|
|
820
|
-
output.logs = limited_buffer.buffer.Bytes()
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
// Writes the debug logs and parsed stack trace to a file on disk.
|
|
824
|
-
func create_issue_file(issue_file_name string, output *vopr_output, message_hash []byte) {
|
|
825
|
-
full_output := append(output.logs, output.stack_trace...)
|
|
826
|
-
error := os.WriteFile(issue_file_name, full_output, 0666)
|
|
827
|
-
if error != nil {
|
|
828
|
-
error_message := fmt.Sprintf("Failed to write to file: %s", error.Error())
|
|
829
|
-
log_error(error_message, message_hash)
|
|
830
|
-
panic(error.Error())
|
|
831
|
-
} else {
|
|
832
|
-
log_message := fmt.Sprintf("Created file: %s", issue_file_name)
|
|
833
|
-
log_info(log_message, message_hash)
|
|
834
|
-
}
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
// Submits a GitHub issue that includes the debug logs and parsed stack trace.
|
|
838
|
-
func create_github_issue(message vopr_message, output *vopr_output) error {
|
|
839
|
-
body := output.create_issue_markdown(message)
|
|
840
|
-
if output.seed_passed {
|
|
841
|
-
body = "Note this seed passed when it was rerun by the VOPR Hub.<br><br>" + body
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
bug_string := get_bug_name(message.bug)
|
|
845
|
-
title := fmt.Sprintf(
|
|
846
|
-
"%s%s: %d",
|
|
847
|
-
strings.ToUpper(string(bug_string[0])),
|
|
848
|
-
string(bug_string[1:]),
|
|
849
|
-
message.seed,
|
|
850
|
-
)
|
|
851
|
-
|
|
852
|
-
issue_contents, error := json.Marshal(struct {
|
|
853
|
-
Title string `json:"title"`
|
|
854
|
-
Body string `json:"body"`
|
|
855
|
-
Labels []string `json:"labels"`
|
|
856
|
-
}{
|
|
857
|
-
Title: title,
|
|
858
|
-
Body: body,
|
|
859
|
-
Labels: []string{"seed", bug_string},
|
|
860
|
-
})
|
|
861
|
-
|
|
862
|
-
if error != nil {
|
|
863
|
-
return error
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
issue := strings.NewReader(string(issue_contents))
|
|
867
|
-
post_request, error := http.NewRequest(
|
|
868
|
-
"POST",
|
|
869
|
-
repository_url+"/issues",
|
|
870
|
-
issue,
|
|
871
|
-
)
|
|
872
|
-
if error != nil {
|
|
873
|
-
log_error("Failed to create the HTTP request for the GitHub API", message.hash[:])
|
|
874
|
-
return error
|
|
875
|
-
}
|
|
876
|
-
post_request.Header.Set("Authorization", "token "+developer_token)
|
|
877
|
-
|
|
878
|
-
post_response, error := http.DefaultClient.Do(post_request)
|
|
879
|
-
if error != nil {
|
|
880
|
-
log_error("Failed to send the HTTP request for the GitHub API", message.hash[:])
|
|
881
|
-
return error
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
defer post_response.Body.Close()
|
|
885
|
-
|
|
886
|
-
if post_response.StatusCode < 200 || post_response.StatusCode > 299 {
|
|
887
|
-
error_message := fmt.Sprintf(
|
|
888
|
-
"Received a non 2xx status code from GitHub. StatusCode: %d. Status: %s",
|
|
889
|
-
post_response.StatusCode,
|
|
890
|
-
post_response.Status,
|
|
891
|
-
)
|
|
892
|
-
log_error("Failed to get a reply from the GitHub API", message.hash[:])
|
|
893
|
-
return fmt.Errorf(error_message)
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
log_info(
|
|
897
|
-
fmt.Sprintf(
|
|
898
|
-
"GitHub issue has been created. Received StatusCode: %d and Status: %s.",
|
|
899
|
-
post_response.StatusCode,
|
|
900
|
-
post_response.Status,
|
|
901
|
-
),
|
|
902
|
-
message.hash[:],
|
|
903
|
-
)
|
|
904
|
-
return nil
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
// Escape characters that have special use in Markdown.
|
|
908
|
-
func make_markdown_compatible(text string) string {
|
|
909
|
-
text = strings.ReplaceAll(text, "^", "")
|
|
910
|
-
text = strings.ReplaceAll(text, "\"", "\\\"")
|
|
911
|
-
text = strings.ReplaceAll(text, "\t", "")
|
|
912
|
-
return text
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
func load_environment_variable(name string) string {
|
|
916
|
-
val, _ := os.LookupEnv(name)
|
|
917
|
-
if val == "" {
|
|
918
|
-
log_error(fmt.Sprintf("env %s not set", name), nil)
|
|
919
|
-
usage()
|
|
920
|
-
os.Exit(1)
|
|
921
|
-
} else if name == "DEVELOPER_TOKEN" {
|
|
922
|
-
log_debug(fmt.Sprintf("env %s=*****", name), nil)
|
|
923
|
-
} else {
|
|
924
|
-
log_debug(fmt.Sprintf("env %s=%q", name, val), nil)
|
|
925
|
-
}
|
|
926
|
-
return val
|
|
927
|
-
}
|
|
928
|
-
|
|
929
|
-
// Ensures that a variable isn't empty or only white space.
|
|
930
|
-
func not_empty(variable *string) bool {
|
|
931
|
-
*variable = strings.TrimSpace(*variable)
|
|
932
|
-
if *variable == "" {
|
|
933
|
-
return false
|
|
934
|
-
} else {
|
|
935
|
-
return true
|
|
936
|
-
}
|
|
937
|
-
}
|
|
938
|
-
|
|
939
|
-
func log_error(message string, vopr_message_hash []byte) {
|
|
940
|
-
log_message("error: ", message, vopr_message_hash)
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
func log_debug(message string, vopr_message_hash []byte) {
|
|
944
|
-
if debug_mode {
|
|
945
|
-
log_message("debug: ", message, vopr_message_hash)
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
func log_info(message string, vopr_message_hash []byte) {
|
|
950
|
-
log_message("info: ", message, vopr_message_hash)
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
// Formats all the log messages and adds a timestamp to them.
|
|
954
|
-
func log_message(log_level string, message string, vopr_message_hash []byte) {
|
|
955
|
-
// Gets the current time in UTC.
|
|
956
|
-
timestamp := time.Now().UTC().Format(time.RFC3339)
|
|
957
|
-
if vopr_message_hash != nil {
|
|
958
|
-
// Only use the first 16 bytes of the message hash as the ID.
|
|
959
|
-
fmt.Printf("%s %sID:%x %s\n", timestamp, log_level, vopr_message_hash, message)
|
|
960
|
-
} else {
|
|
961
|
-
fmt.Printf("%s %s%s\n", timestamp, log_level, message)
|
|
962
|
-
}
|
|
963
|
-
}
|
|
964
|
-
|
|
965
|
-
func main() {
|
|
966
|
-
// Determine the mode in which to run the VOPR Hub.
|
|
967
|
-
flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
|
|
968
|
-
flags.BoolVar(&debug_mode, "debug", false, "runs with debugging logs enabled")
|
|
969
|
-
if err := flags.Parse(os.Args[1:]); err != nil {
|
|
970
|
-
usage()
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
// Retrieve all the required environment variables up front.
|
|
974
|
-
tigerbeetle_directory = load_environment_variable("TIGERBEETLE_DIRECTORY")
|
|
975
|
-
issue_directory = load_environment_variable("ISSUE_DIRECTORY")
|
|
976
|
-
developer_token = load_environment_variable("DEVELOPER_TOKEN")
|
|
977
|
-
vopr_hub_address = load_environment_variable("VOPR_HUB_ADDRESS")
|
|
978
|
-
repository_url = load_environment_variable("REPOSITORY_URL")
|
|
979
|
-
|
|
980
|
-
// This channel ensures no more than max_concurrent_connections are being handled at one time.
|
|
981
|
-
track_connections := make(chan bool, max_concurrent_connections)
|
|
982
|
-
|
|
983
|
-
// The channel will receieve fixed-size byte arrays from a VOPR.
|
|
984
|
-
vopr_message_channel := make(chan vopr_message, max_queuing_messages)
|
|
985
|
-
|
|
986
|
-
// Create a worker Goroutine to call process on each item as it appears in the channel.
|
|
987
|
-
go worker(vopr_message_channel)
|
|
988
|
-
|
|
989
|
-
listener, error := net.Listen("tcp", vopr_hub_address)
|
|
990
|
-
if error != nil {
|
|
991
|
-
error_message := fmt.Sprintf("Could not listen on %s: %s", vopr_hub_address, error.Error())
|
|
992
|
-
log_error(error_message, nil)
|
|
993
|
-
os.Exit(1)
|
|
994
|
-
}
|
|
995
|
-
defer listener.Close()
|
|
996
|
-
log_info("Listening...", nil)
|
|
997
|
-
|
|
998
|
-
for {
|
|
999
|
-
connection, error := listener.Accept()
|
|
1000
|
-
connection.SetReadDeadline(time.Now().Add(10 * time.Second))
|
|
1001
|
-
if error != nil {
|
|
1002
|
-
error_message := fmt.Sprintf(
|
|
1003
|
-
"Unable to set read deadline on the connection: %s",
|
|
1004
|
-
error.Error(),
|
|
1005
|
-
)
|
|
1006
|
-
log_error(error_message, nil)
|
|
1007
|
-
connection.Close()
|
|
1008
|
-
continue
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
select {
|
|
1012
|
-
case track_connections <- true:
|
|
1013
|
-
go handle_connection(track_connections, connection, vopr_message_channel)
|
|
1014
|
-
default:
|
|
1015
|
-
log_info(
|
|
1016
|
-
"Connection closed because there are currently too many open connections",
|
|
1017
|
-
nil,
|
|
1018
|
-
)
|
|
1019
|
-
connection.Close()
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
|
-
}
|