relay-flow 0.2.4-alpha → 0.2.5-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -15
- package/cmd/relay-flow/backend_selection_test.go +149 -0
- package/cmd/relay-flow/main.go +94 -13
- package/cmd/relay-flow/scenario_test.go +19 -2
- package/cmd/relay-flow/serve.go +98 -19
- package/cmd/relay-flow/serve_recovery_test.go +100 -0
- package/cmd/relay-flow/temporal_init.go +170 -0
- package/cmd/relay-flow/temporal_init_test.go +217 -0
- package/cmd/relay-flow/temporal_report_test.go +733 -0
- package/examples/config-reference.yaml +2 -2
- package/examples/minimal-beads-task-workflow.yaml +2 -1
- package/examples/workflow-reference.yaml +2 -1
- package/go.mod +37 -16
- package/go.sum +129 -61
- package/internal/config/machine.go +33 -1
- package/internal/config/machine_test.go +76 -0
- package/internal/execution/goworkflows/engine.go +13 -38
- package/internal/execution/goworkflows/projection.go +47 -464
- package/internal/execution/projection/projection.go +867 -0
- package/internal/execution/projection/projection_test.go +347 -0
- package/internal/execution/temporal/activities.go +567 -0
- package/internal/execution/temporal/engine.go +384 -0
- package/internal/execution/temporal/engine_test.go +277 -0
- package/internal/execution/temporal/interpreter.go +736 -0
- package/internal/execution/temporal/operations.go +455 -0
- package/internal/execution/temporal/operations_test.go +101 -0
- package/internal/execution/temporal/recovery.go +194 -0
- package/internal/execution/temporal/recovery_runtime.go +41 -0
- package/internal/execution/temporal/recovery_test.go +102 -0
- package/internal/execution/temporal/snapshot_restart_test.go +72 -0
- package/internal/execution/temporal/spike_test.go +934 -0
- package/internal/execution/temporal/visibility_lag_test.go +415 -0
- package/internal/harness/opencode/opencode.go +3 -1
- package/internal/harness/opencode/opencode_test.go +1 -1
- package/internal/harness/opencode/repo_setup.go +1 -1
- package/internal/harness/pi/pi.go +49 -46
- package/internal/harness/pi/pi_test.go +26 -10
- package/internal/harness/pi/prompt_test.go +30 -1
- package/internal/harness/pi/validation_test.go +27 -51
- package/internal/runner/herdr/herdr.go +14 -0
- package/internal/runner/herdr/herdr_test.go +20 -0
- package/internal/runner/orca/orca.go +33 -0
- package/internal/runner/orca/orca_test.go +33 -4
- package/internal/runner/runner.go +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -354,37 +354,42 @@ lower-scope input.
|
|
|
354
354
|
|
|
355
355
|
Pi has one built-in coding agent. Relay-flow keeps Pi's configured model,
|
|
356
356
|
provider, tools, extensions, and settings, while allowing a workflow node to
|
|
357
|
-
select a
|
|
358
|
-
coding agent without an additional
|
|
359
|
-
`coder` or `reviewer`
|
|
360
|
-
`.pi/
|
|
361
|
-
|
|
357
|
+
select a project-owned prompt template. `agent: default` uses Pi's built-in
|
|
358
|
+
coding agent without an additional prompt template. A non-default value such as
|
|
359
|
+
`coder` or `reviewer` selects Pi's native project template
|
|
360
|
+
`.pi/prompts/<agent>.md` and submits the rendered task text as the template's
|
|
361
|
+
arguments with `/<agent> ...`.
|
|
362
362
|
|
|
363
|
-
Pi
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
363
|
+
Pi owns prompt-template discovery and expansion. Relay-flow does not maintain a
|
|
364
|
+
named-agent registry or inspect template files; it only validates the safe
|
|
365
|
+
template command name and passes Pi the corresponding
|
|
366
|
+
`--prompt-template .pi/prompts/<agent>.md` resource. The workflow agent value is
|
|
367
|
+
never treated as a model ID and relay-flow never passes an OpenCode-style
|
|
368
|
+
`--agent` option.
|
|
367
369
|
|
|
368
|
-
For example, a Pi workflow with
|
|
370
|
+
For example, a Pi workflow with project prompt templates uses:
|
|
369
371
|
|
|
370
372
|
```yaml
|
|
371
373
|
type: agent
|
|
372
374
|
agent: coder
|
|
373
375
|
```
|
|
374
376
|
|
|
375
|
-
with:
|
|
377
|
+
with project templates:
|
|
376
378
|
|
|
377
379
|
```text
|
|
378
|
-
.pi/
|
|
379
|
-
.pi/
|
|
380
|
+
.pi/prompts/coder.md
|
|
381
|
+
.pi/prompts/reviewer.md
|
|
380
382
|
```
|
|
381
383
|
|
|
382
|
-
Pi node launches use the installed Pi
|
|
384
|
+
Pi node launches use the installed Pi interactive command contract:
|
|
383
385
|
|
|
384
386
|
```text
|
|
385
|
-
pi --name <ticket>:<node> [--
|
|
387
|
+
pi --name <ticket>:<node> [--prompt-template .pi/prompts/<agent>.md] [--session-id <persisted-session-id>] <prompt>
|
|
386
388
|
```
|
|
387
389
|
|
|
390
|
+
For a named agent, `<prompt>` is `/agent <rendered relay-flow task or
|
|
391
|
+
feedback prompt>`; `default` keeps the rendered prompt unchanged.
|
|
392
|
+
|
|
388
393
|
The prompt is one positional argument. Pi 0.84.1 rejects a bare `--`, so the
|
|
389
394
|
launch command does not include one. A persisted session uses
|
|
390
395
|
`--session-id`; print mode, JSON/RPC mode, and extension-install flags are not
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
package main
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"database/sql"
|
|
6
|
+
"fmt"
|
|
7
|
+
"os"
|
|
8
|
+
"path/filepath"
|
|
9
|
+
"testing"
|
|
10
|
+
"time"
|
|
11
|
+
|
|
12
|
+
"github.com/rajpopat27/relay-flow/internal/config"
|
|
13
|
+
"github.com/rajpopat27/relay-flow/internal/execution/projection"
|
|
14
|
+
"github.com/rajpopat27/relay-flow/internal/paths"
|
|
15
|
+
"github.com/rajpopat27/relay-flow/internal/server"
|
|
16
|
+
_ "modernc.org/sqlite"
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
// These integration tests are deliberately live-gated because they exercise
|
|
20
|
+
// the selected durable backend and the local Temporal service. They use the
|
|
21
|
+
// same fake task/runner/harness factories as the composition scenario, but no
|
|
22
|
+
// task-system state or workflow is needed to prove worker selection.
|
|
23
|
+
func TestServeSelectsConfiguredTemporalExecutor(t *testing.T) {
|
|
24
|
+
if os.Getenv("RELAY_FLOW_TEMPORAL_LIVE") != "1" {
|
|
25
|
+
t.Skip("set RELAY_FLOW_TEMPORAL_LIVE=1 to run backend selection against Temporal")
|
|
26
|
+
}
|
|
27
|
+
assertServeUsesTemporal(t, false)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
func TestServeRecoverSelectsConfiguredTemporalExecutor(t *testing.T) {
|
|
31
|
+
if os.Getenv("RELAY_FLOW_TEMPORAL_LIVE") != "1" {
|
|
32
|
+
t.Skip("set RELAY_FLOW_TEMPORAL_LIVE=1 to run backend recovery against Temporal")
|
|
33
|
+
}
|
|
34
|
+
assertServeUsesTemporal(t, true)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
func assertServeUsesTemporal(t *testing.T, recover bool) {
|
|
38
|
+
t.Helper()
|
|
39
|
+
log := newScenarioLog()
|
|
40
|
+
setScenarioFactoryAdapters(newScenarioTaskSystem(log), newScenarioRunner(log), newScenarioHarness(log))
|
|
41
|
+
|
|
42
|
+
root := filepath.Join(t.TempDir(), ".relay-flow")
|
|
43
|
+
t.Setenv("RELAY_FLOW_HOME", root)
|
|
44
|
+
p := pathsForRoot(root)
|
|
45
|
+
if err := paths.Ensure(p); err != nil {
|
|
46
|
+
t.Fatal(err)
|
|
47
|
+
}
|
|
48
|
+
namespace := fmt.Sprintf("relay-flow-serve-%d", time.Now().UnixNano())
|
|
49
|
+
if err := ensureTemporalNamespace(context.Background(), "localhost:7233", namespace, 30); err != nil {
|
|
50
|
+
t.Fatal(err)
|
|
51
|
+
}
|
|
52
|
+
// Namespace registration is acknowledged before every frontend worker
|
|
53
|
+
// necessarily observes it.
|
|
54
|
+
time.Sleep(5 * time.Second)
|
|
55
|
+
if err := config.SaveMachine(p.Config, &config.Machine{
|
|
56
|
+
TaskPlugin: scenarioTaskPlugin,
|
|
57
|
+
RunnerPlugin: scenarioRunnerPlugin,
|
|
58
|
+
HarnessPlugin: scenarioHarnessPlugin,
|
|
59
|
+
ExecutorPlugin: "temporal",
|
|
60
|
+
TemporalAddress: "localhost:7233",
|
|
61
|
+
TemporalNamespace: namespace,
|
|
62
|
+
CompletedRunRetentionDays: 30,
|
|
63
|
+
}); err != nil {
|
|
64
|
+
t.Fatal(err)
|
|
65
|
+
}
|
|
66
|
+
if err := projection.InitDatabaseWithIdentity(p.Database, projection.ExecutorIdentity{
|
|
67
|
+
ExecutorPlugin: "temporal", TemporalAddress: "localhost:7233", TemporalNamespace: namespace,
|
|
68
|
+
}); err != nil {
|
|
69
|
+
t.Fatal(err)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
ctx, cancel := context.WithCancel(context.Background())
|
|
73
|
+
defer cancel()
|
|
74
|
+
done := make(chan error, 1)
|
|
75
|
+
go func() { done <- serveRoot(ctx, p, recover) }()
|
|
76
|
+
client := server.NewClient(p.Socket)
|
|
77
|
+
waitForServerOrError(t, client, done)
|
|
78
|
+
|
|
79
|
+
assertNoEmbeddedExecutionTables(t, p.Database)
|
|
80
|
+
if err := client.Stop(context.Background()); err != nil {
|
|
81
|
+
t.Fatal(err)
|
|
82
|
+
}
|
|
83
|
+
select {
|
|
84
|
+
case err := <-done:
|
|
85
|
+
if err != nil {
|
|
86
|
+
t.Fatalf("serveRoot(%v): %v", recover, err)
|
|
87
|
+
}
|
|
88
|
+
case <-time.After(15 * time.Second):
|
|
89
|
+
t.Fatalf("serveRoot(%v) did not stop", recover)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
func waitForServerOrError(t *testing.T, client *server.Client, done <-chan error) {
|
|
94
|
+
t.Helper()
|
|
95
|
+
deadline := time.Now().Add(20 * time.Second)
|
|
96
|
+
for time.Now().Before(deadline) {
|
|
97
|
+
select {
|
|
98
|
+
case err := <-done:
|
|
99
|
+
t.Fatalf("serveRoot exited before socket became ready: %v", err)
|
|
100
|
+
default:
|
|
101
|
+
}
|
|
102
|
+
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
103
|
+
_, err := client.ListRepos(ctx)
|
|
104
|
+
cancel()
|
|
105
|
+
if err == nil {
|
|
106
|
+
return
|
|
107
|
+
}
|
|
108
|
+
time.Sleep(20 * time.Millisecond)
|
|
109
|
+
}
|
|
110
|
+
select {
|
|
111
|
+
case err := <-done:
|
|
112
|
+
t.Fatalf("serveRoot exited before socket became ready: %v", err)
|
|
113
|
+
default:
|
|
114
|
+
t.Fatal("server did not become ready")
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
func assertNoEmbeddedExecutionTables(t *testing.T, databasePath string) {
|
|
119
|
+
t.Helper()
|
|
120
|
+
db, err := sql.Open("sqlite", databasePath)
|
|
121
|
+
if err != nil {
|
|
122
|
+
t.Fatal(err)
|
|
123
|
+
}
|
|
124
|
+
defer db.Close()
|
|
125
|
+
var count int
|
|
126
|
+
if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('instances', 'history_events', 'activity_tasks')`).Scan(&count); err != nil {
|
|
127
|
+
t.Fatal(err)
|
|
128
|
+
}
|
|
129
|
+
if count != 0 {
|
|
130
|
+
t.Fatalf("Temporal projection contains embedded go-workflows tables: %d", count)
|
|
131
|
+
}
|
|
132
|
+
var plugin, address, namespace string
|
|
133
|
+
if err := db.QueryRow(`SELECT executor_plugin, temporal_address, temporal_namespace FROM relay_executor_identity WHERE singleton = 1`).Scan(&plugin, &address, &namespace); err != nil {
|
|
134
|
+
t.Fatal(err)
|
|
135
|
+
}
|
|
136
|
+
if plugin != "temporal" || address != "localhost:7233" || namespace == "" {
|
|
137
|
+
t.Fatalf("executor identity = %q/%q/%q", plugin, address, namespace)
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
func pathsForRoot(root string) paths.Paths {
|
|
142
|
+
return paths.Paths{
|
|
143
|
+
Root: root, Config: filepath.Join(root, "config.yaml"),
|
|
144
|
+
Credentials: filepath.Join(root, "credentials.yaml"), Workflows: filepath.Join(root, "workflows"),
|
|
145
|
+
Database: filepath.Join(root, "state.db"), Socket: filepath.Join(root, "server.sock"),
|
|
146
|
+
Lock: filepath.Join(root, "server.lock"), ServerLog: filepath.Join(root, "server.log"),
|
|
147
|
+
PluginLog: filepath.Join(root, "plugin.log"),
|
|
148
|
+
}
|
|
149
|
+
}
|
package/cmd/relay-flow/main.go
CHANGED
|
@@ -24,7 +24,7 @@ import (
|
|
|
24
24
|
"github.com/charmbracelet/huh"
|
|
25
25
|
"github.com/mattn/go-isatty"
|
|
26
26
|
"github.com/rajpopat27/relay-flow/internal/config"
|
|
27
|
-
"github.com/rajpopat27/relay-flow/internal/execution/
|
|
27
|
+
"github.com/rajpopat27/relay-flow/internal/execution/projection"
|
|
28
28
|
"github.com/rajpopat27/relay-flow/internal/harness"
|
|
29
29
|
"github.com/rajpopat27/relay-flow/internal/logging"
|
|
30
30
|
"github.com/rajpopat27/relay-flow/internal/paths"
|
|
@@ -134,6 +134,8 @@ func usage(w io.Writer) {
|
|
|
134
134
|
|
|
135
135
|
Usage:
|
|
136
136
|
relay-flow init [--force] [--task-plugin <name> --runner-plugin <name> --harness-plugin <name>]
|
|
137
|
+
[--executor-plugin <goworkflows|temporal>]
|
|
138
|
+
[--temporal-address <host:port>] [--temporal-namespace <name>]
|
|
137
139
|
relay-flow task auth [task-plugin options]
|
|
138
140
|
relay-flow serve [--recover] [--debug] [--background]
|
|
139
141
|
relay-flow stop
|
|
@@ -170,6 +172,9 @@ func cmdInit(p paths.Paths, args []string, stdin io.Reader) int {
|
|
|
170
172
|
taskName := fs.String("task-plugin", "", "task plugin name (non-interactive)")
|
|
171
173
|
runnerName := fs.String("runner-plugin", "", "runner plugin name (non-interactive)")
|
|
172
174
|
harnessName := fs.String("harness-plugin", "", "harness plugin name (non-interactive)")
|
|
175
|
+
executorName := fs.String("executor-plugin", "", "durable executor name (goworkflows or temporal)")
|
|
176
|
+
temporalAddress := fs.String("temporal-address", "", "Temporal server address")
|
|
177
|
+
temporalNamespace := fs.String("temporal-namespace", "", "Temporal namespace/team name")
|
|
173
178
|
force := fs.Bool("force", false, "update plugin selections while preserving existing state")
|
|
174
179
|
if err := fs.Parse(args); err != nil {
|
|
175
180
|
return exitUsage
|
|
@@ -215,7 +220,7 @@ func cmdInit(p paths.Paths, args []string, stdin io.Reader) int {
|
|
|
215
220
|
}
|
|
216
221
|
defer unlock()
|
|
217
222
|
if databaseExists {
|
|
218
|
-
active, err :=
|
|
223
|
+
active, err := projection.HasNonterminalRuns(p.Database)
|
|
219
224
|
if err != nil {
|
|
220
225
|
fmt.Fprintln(os.Stderr, "init: "+err.Error())
|
|
221
226
|
return exitFail
|
|
@@ -227,9 +232,11 @@ func cmdInit(p paths.Paths, args []string, stdin io.Reader) int {
|
|
|
227
232
|
}
|
|
228
233
|
}
|
|
229
234
|
|
|
230
|
-
// Selection precedence: flags → TTY form → stdin lines. The
|
|
231
|
-
//
|
|
235
|
+
// Selection precedence: flags → TTY form → legacy stdin lines. The
|
|
236
|
+
// three-line stdin path remains embedded-mode input and never consumes
|
|
237
|
+
// Temporal answers.
|
|
232
238
|
var names []string
|
|
239
|
+
executor := strings.TrimSpace(*executorName)
|
|
233
240
|
switch {
|
|
234
241
|
case flagged:
|
|
235
242
|
names = []string{*taskName, *runnerName, *harnessName}
|
|
@@ -240,6 +247,10 @@ func cmdInit(p paths.Paths, args []string, stdin io.Reader) int {
|
|
|
240
247
|
fmt.Fprintln(os.Stderr, "init: "+err.Error())
|
|
241
248
|
return exitFail
|
|
242
249
|
}
|
|
250
|
+
if len(names) == 4 {
|
|
251
|
+
executor = names[3]
|
|
252
|
+
names = names[:3]
|
|
253
|
+
}
|
|
243
254
|
default:
|
|
244
255
|
names = readInitLines(stdin)
|
|
245
256
|
if names == nil {
|
|
@@ -247,6 +258,22 @@ func cmdInit(p paths.Paths, args []string, stdin io.Reader) int {
|
|
|
247
258
|
return exitFail
|
|
248
259
|
}
|
|
249
260
|
}
|
|
261
|
+
if executor == executorTemporal && isTTY(stdin) {
|
|
262
|
+
address, namespace, err := promptTemporalSettings(*temporalAddress, *temporalNamespace)
|
|
263
|
+
if err != nil {
|
|
264
|
+
fmt.Fprintln(os.Stderr, "init: "+err.Error())
|
|
265
|
+
return exitFail
|
|
266
|
+
}
|
|
267
|
+
*temporalAddress = address
|
|
268
|
+
*temporalNamespace = namespace
|
|
269
|
+
}
|
|
270
|
+
if executor == "" {
|
|
271
|
+
executor = executorGoworkflows
|
|
272
|
+
}
|
|
273
|
+
if executor != executorGoworkflows && executor != executorTemporal {
|
|
274
|
+
fmt.Fprintln(os.Stderr, "init: unknown executor plugin "+executor)
|
|
275
|
+
return exitUsage
|
|
276
|
+
}
|
|
250
277
|
// Validate against the registered factories; unknown names list the
|
|
251
278
|
// registered set per design (no silent acceptance).
|
|
252
279
|
if err := task.ValidateName(names[0]); err != nil {
|
|
@@ -262,13 +289,42 @@ func cmdInit(p paths.Paths, args []string, stdin io.Reader) int {
|
|
|
262
289
|
return exitFail
|
|
263
290
|
}
|
|
264
291
|
|
|
265
|
-
cfg := &config.Machine{
|
|
292
|
+
cfg := &config.Machine{
|
|
293
|
+
PollIntervalSeconds: 15, CompletedRunRetentionDays: 30,
|
|
294
|
+
KeepTerminalsAlive: true, KeepSessionsAlive: true,
|
|
295
|
+
ExecutorPlugin: executor,
|
|
296
|
+
}
|
|
266
297
|
if *force && configExists {
|
|
267
298
|
cfg, err = config.LoadMachine(p.Config)
|
|
268
299
|
if err != nil {
|
|
269
300
|
fmt.Fprintln(os.Stderr, err)
|
|
270
301
|
return exitFail
|
|
271
302
|
}
|
|
303
|
+
if strings.TrimSpace(*executorName) == "" {
|
|
304
|
+
executor = cfg.ExecutorPlugin
|
|
305
|
+
}
|
|
306
|
+
cfg.ExecutorPlugin = executor
|
|
307
|
+
}
|
|
308
|
+
if executor == executorTemporal {
|
|
309
|
+
if strings.TrimSpace(*temporalAddress) != "" {
|
|
310
|
+
cfg.TemporalAddress = strings.TrimSpace(*temporalAddress)
|
|
311
|
+
} else if cfg.TemporalAddress == "" {
|
|
312
|
+
cfg.TemporalAddress = defaultTemporalAddr
|
|
313
|
+
}
|
|
314
|
+
if strings.TrimSpace(*temporalNamespace) != "" {
|
|
315
|
+
cfg.TemporalNamespace = strings.TrimSpace(*temporalNamespace)
|
|
316
|
+
}
|
|
317
|
+
if cfg.TemporalNamespace == "" {
|
|
318
|
+
fmt.Fprintln(os.Stderr, "init: --temporal-namespace is required for executor-plugin temporal")
|
|
319
|
+
return exitUsage
|
|
320
|
+
}
|
|
321
|
+
} else {
|
|
322
|
+
if strings.TrimSpace(*temporalAddress) != "" || strings.TrimSpace(*temporalNamespace) != "" {
|
|
323
|
+
fmt.Fprintln(os.Stderr, "init: Temporal address/namespace require executor-plugin temporal")
|
|
324
|
+
return exitUsage
|
|
325
|
+
}
|
|
326
|
+
cfg.TemporalAddress = ""
|
|
327
|
+
cfg.TemporalNamespace = ""
|
|
272
328
|
}
|
|
273
329
|
cfg.TaskPlugin = names[0]
|
|
274
330
|
cfg.RunnerPlugin = names[1]
|
|
@@ -297,17 +353,37 @@ func cmdInit(p paths.Paths, args []string, stdin io.Reader) int {
|
|
|
297
353
|
} else {
|
|
298
354
|
cfg.HarnessConfig = config.Merge(harnessDefaults, cfg.HarnessConfig)
|
|
299
355
|
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
356
|
+
identity := projection.ExecutorIdentity{
|
|
357
|
+
ExecutorPlugin: executor,
|
|
358
|
+
TemporalAddress: cfg.TemporalAddress,
|
|
359
|
+
TemporalNamespace: cfg.TemporalNamespace,
|
|
360
|
+
}
|
|
361
|
+
// Force mode verifies the immutable durable identity before contacting
|
|
362
|
+
// Temporal or replacing configuration. Legacy marker-less homes may only
|
|
363
|
+
// be adopted by goworkflows.
|
|
364
|
+
if databaseExists {
|
|
365
|
+
if err := verifyExecutorIdentity(p.Database, identity); err != nil {
|
|
366
|
+
fmt.Fprintln(os.Stderr, "init: "+err.Error())
|
|
367
|
+
return exitFail
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
if executor == executorTemporal {
|
|
371
|
+
if err := ensureTemporalNamespace(context.Background(), cfg.TemporalAddress, cfg.TemporalNamespace, cfg.CompletedRunRetentionDays); err != nil {
|
|
372
|
+
fmt.Fprintln(os.Stderr, "init: "+err.Error())
|
|
373
|
+
return exitFail
|
|
374
|
+
}
|
|
303
375
|
}
|
|
304
376
|
if !databaseExists {
|
|
305
|
-
if err :=
|
|
306
|
-
fmt.Fprintln(os.Stderr, err)
|
|
377
|
+
if err := projection.InitDatabaseWithIdentity(p.Database, identity); err != nil {
|
|
378
|
+
fmt.Fprintln(os.Stderr, "init: "+err.Error())
|
|
307
379
|
return exitFail
|
|
308
380
|
}
|
|
309
381
|
}
|
|
310
|
-
|
|
382
|
+
if err := config.SaveMachine(p.Config, cfg); err != nil {
|
|
383
|
+
fmt.Fprintln(os.Stderr, err)
|
|
384
|
+
return exitFail
|
|
385
|
+
}
|
|
386
|
+
fmt.Printf("Task system: %s\nRunner: %s\nHarness: %s\nExecutor: %s\nRelay-flow initialized\n", names[0], names[1], names[2], executor)
|
|
311
387
|
return exitOK
|
|
312
388
|
}
|
|
313
389
|
|
|
@@ -345,8 +421,8 @@ func isTTY(stdin io.Reader) bool {
|
|
|
345
421
|
|
|
346
422
|
// pickPluginsInteractive runs one searchable select per plugin type.
|
|
347
423
|
func pickPluginsInteractive() ([]string, error) {
|
|
348
|
-
names := make([]string,
|
|
349
|
-
groups := make([]*huh.Group, 0,
|
|
424
|
+
names := make([]string, 4)
|
|
425
|
+
groups := make([]*huh.Group, 0, 4)
|
|
350
426
|
for _, selection := range []struct {
|
|
351
427
|
title string
|
|
352
428
|
options []string
|
|
@@ -364,6 +440,11 @@ func pickPluginsInteractive() ([]string, error) {
|
|
|
364
440
|
groups = append(groups, huh.NewGroup(field))
|
|
365
441
|
}
|
|
366
442
|
}
|
|
443
|
+
executorField, err := executorSelectField(&names[3])
|
|
444
|
+
if err != nil {
|
|
445
|
+
return nil, err
|
|
446
|
+
}
|
|
447
|
+
groups = append(groups, huh.NewGroup(executorField))
|
|
367
448
|
if len(groups) == 0 {
|
|
368
449
|
return names, nil
|
|
369
450
|
}
|
|
@@ -801,6 +801,8 @@ type scenarioTaskSystem struct {
|
|
|
801
801
|
creates map[string]int
|
|
802
802
|
completions map[string]int
|
|
803
803
|
completeFailures int
|
|
804
|
+
recoveryTickets []task.Ticket
|
|
805
|
+
recoveryPolls int
|
|
804
806
|
}
|
|
805
807
|
|
|
806
808
|
var _ task.System = (*scenarioTaskSystem)(nil)
|
|
@@ -816,11 +818,19 @@ func newScenarioTaskSystem(log *scenarioLog) *scenarioTaskSystem {
|
|
|
816
818
|
func (s *scenarioTaskSystem) Poll(context.Context) ([]task.Ticket, error) {
|
|
817
819
|
s.log.add("poll")
|
|
818
820
|
s.mu.Lock()
|
|
819
|
-
defer s.mu.Unlock()
|
|
820
821
|
if s.claimed {
|
|
822
|
+
if s.recoveryPolls > 0 {
|
|
823
|
+
s.recoveryPolls--
|
|
824
|
+
tickets := append([]task.Ticket(nil), s.recoveryTickets...)
|
|
825
|
+
s.mu.Unlock()
|
|
826
|
+
return tickets, nil
|
|
827
|
+
}
|
|
828
|
+
s.mu.Unlock()
|
|
821
829
|
return nil, nil
|
|
822
830
|
}
|
|
823
|
-
|
|
831
|
+
tickets := []task.Ticket{{ID: "ticket-1", Key: scenarioTicket, Title: "Scenario ticket"}}
|
|
832
|
+
s.mu.Unlock()
|
|
833
|
+
return tickets, nil
|
|
824
834
|
}
|
|
825
835
|
|
|
826
836
|
func (s *scenarioTaskSystem) CompileFilter(config.RawValues) (func(task.Ticket) bool, error) {
|
|
@@ -958,6 +968,13 @@ func (s *scenarioTaskSystem) setCompleteFailures(n int) {
|
|
|
958
968
|
s.mu.Unlock()
|
|
959
969
|
}
|
|
960
970
|
|
|
971
|
+
func (s *scenarioTaskSystem) setRecoveryTickets(tickets []task.Ticket, polls int) {
|
|
972
|
+
s.mu.Lock()
|
|
973
|
+
s.recoveryTickets = append([]task.Ticket(nil), tickets...)
|
|
974
|
+
s.recoveryPolls = polls
|
|
975
|
+
s.mu.Unlock()
|
|
976
|
+
}
|
|
977
|
+
|
|
961
978
|
func (s *scenarioTaskSystem) mailboxCreateCount(node string) int {
|
|
962
979
|
s.mu.Lock()
|
|
963
980
|
defer s.mu.Unlock()
|
package/cmd/relay-flow/serve.go
CHANGED
|
@@ -21,6 +21,8 @@ import (
|
|
|
21
21
|
|
|
22
22
|
"github.com/rajpopat27/relay-flow/internal/config"
|
|
23
23
|
"github.com/rajpopat27/relay-flow/internal/execution/goworkflows"
|
|
24
|
+
"github.com/rajpopat27/relay-flow/internal/execution/projection"
|
|
25
|
+
temporalexec "github.com/rajpopat27/relay-flow/internal/execution/temporal"
|
|
24
26
|
"github.com/rajpopat27/relay-flow/internal/harness"
|
|
25
27
|
"github.com/rajpopat27/relay-flow/internal/paths"
|
|
26
28
|
recoverpkg "github.com/rajpopat27/relay-flow/internal/recover"
|
|
@@ -179,13 +181,35 @@ func serveRoot(ctx context.Context, p paths.Paths, recover bool) error {
|
|
|
179
181
|
// provably empty database, and Engine.Start has no prior history
|
|
180
182
|
// to resume.
|
|
181
183
|
if recover {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
+
// Validate the existing installation identity before moving any state
|
|
185
|
+
// aside. A changed executor/address/namespace must fail closed; only a
|
|
186
|
+
// genuinely missing database may be initialized by recovery.
|
|
187
|
+
if _, statErr := os.Stat(p.Database); statErr == nil {
|
|
188
|
+
expected := projection.ExecutorIdentity{ExecutorPlugin: cfg.ExecutorPlugin}
|
|
189
|
+
if cfg.ExecutorPlugin == executorTemporal {
|
|
190
|
+
expected.TemporalAddress = cfg.TemporalAddress
|
|
191
|
+
expected.TemporalNamespace = cfg.TemporalNamespace
|
|
192
|
+
}
|
|
193
|
+
if err := verifyExecutorIdentity(p.Database, expected); err != nil && !errors.Is(err, ErrProjectionUnusable) {
|
|
194
|
+
return fmt.Errorf("recover: verify executor identity before backup: %w", err)
|
|
195
|
+
}
|
|
196
|
+
} else if !os.IsNotExist(statErr) {
|
|
197
|
+
return fmt.Errorf("recover: stat database %s: %w", p.Database, statErr)
|
|
198
|
+
}
|
|
199
|
+
stamp := time.Now().UTC().Format("20060102T150405.000000000Z")
|
|
200
|
+
suffixes := []string{"", "-wal", "-shm"}
|
|
201
|
+
backupStem, err := recoveryBackupStem(p.Database, stamp, suffixes)
|
|
202
|
+
if err != nil {
|
|
203
|
+
return err
|
|
204
|
+
}
|
|
205
|
+
for _, suffix := range suffixes {
|
|
184
206
|
src := p.Database + suffix
|
|
185
207
|
if _, err := os.Stat(src); os.IsNotExist(err) {
|
|
186
208
|
continue
|
|
209
|
+
} else if err != nil {
|
|
210
|
+
return fmt.Errorf("inspect stale database %s: %w", src, err)
|
|
187
211
|
}
|
|
188
|
-
dst :=
|
|
212
|
+
dst := backupStem + ".bak" + suffix
|
|
189
213
|
if err := os.Rename(src, dst); err != nil {
|
|
190
214
|
return fmt.Errorf("preserve stale database %s as %s: %w", src, dst, err)
|
|
191
215
|
}
|
|
@@ -201,21 +225,32 @@ func serveRoot(ctx context.Context, p paths.Paths, recover bool) error {
|
|
|
201
225
|
}
|
|
202
226
|
}
|
|
203
227
|
|
|
204
|
-
// Open the
|
|
205
|
-
//
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
228
|
+
// Open exactly the configured durable executor. The machine config and
|
|
229
|
+
// projection identity have already selected/fenced this backend; there is
|
|
230
|
+
// deliberately no probe-and-fallback path.
|
|
231
|
+
var engine durableEngine
|
|
232
|
+
runtimePolicy := &runsvc.RuntimePolicy{
|
|
233
|
+
KeepTerminalsAlive: cfg.KeepTerminalsAlive,
|
|
234
|
+
KeepSessionsAlive: cfg.KeepSessionsAlive,
|
|
235
|
+
}
|
|
236
|
+
switch cfg.ExecutorPlugin {
|
|
237
|
+
case "goworkflows":
|
|
238
|
+
engine, err = goworkflows.New(p.Database, goworkflows.Dependencies{
|
|
239
|
+
Repos: repoReg, Runner: rnr, Harness: hrn, TaskSystem: cfg.TaskPlugin,
|
|
240
|
+
RetentionDays: cfg.CompletedRunRetentionDays, Runtime: runtimePolicy,
|
|
241
|
+
})
|
|
242
|
+
case "temporal":
|
|
243
|
+
engine, err = temporalexec.New(p.Database, temporalexec.Dependencies{
|
|
244
|
+
Repos: repoReg, Runner: rnr, Harness: hrn, TaskSystem: cfg.TaskPlugin,
|
|
245
|
+
RetentionDays: cfg.CompletedRunRetentionDays, Runtime: runtimePolicy,
|
|
246
|
+
TemporalAddress: cfg.TemporalAddress, TemporalNamespace: cfg.TemporalNamespace,
|
|
247
|
+
Recover: recover,
|
|
248
|
+
})
|
|
249
|
+
default:
|
|
250
|
+
err = fmt.Errorf("unknown executor plugin %q (want goworkflows or temporal)", cfg.ExecutorPlugin)
|
|
251
|
+
}
|
|
217
252
|
if err != nil {
|
|
218
|
-
return fmt.Errorf("open engine: %w", err)
|
|
253
|
+
return fmt.Errorf("open %s engine: %w", cfg.ExecutorPlugin, err)
|
|
219
254
|
}
|
|
220
255
|
|
|
221
256
|
// 5.2 fail-fast preflight: before workers/pollers start, validate
|
|
@@ -278,7 +313,7 @@ func serveRoot(ctx context.Context, p paths.Paths, recover bool) error {
|
|
|
278
313
|
// recovered runs exist before any poll cycle observes them. Mailbox
|
|
279
314
|
// specs come from the engine so the recover path builds the same
|
|
280
315
|
// description content as normal run execution.
|
|
281
|
-
if recover {
|
|
316
|
+
if recover && cfg.ExecutorPlugin == "goworkflows" {
|
|
282
317
|
specsFor := func(sys task.System, work runsvc.Work, wf *workflow.Workflow) ([]task.MailboxSpec, error) {
|
|
283
318
|
return goworkflows.RenderMailboxSpecs(sys, work, wf)
|
|
284
319
|
}
|
|
@@ -362,6 +397,18 @@ func serveRoot(ctx context.Context, p paths.Paths, recover bool) error {
|
|
|
362
397
|
// single cleanup path.
|
|
363
398
|
stopCtx, stopServe := context.WithCancel(context.Background())
|
|
364
399
|
defer stopServe()
|
|
400
|
+
if fatalEngine, ok := engine.(interface{ FatalErrors() <-chan error }); ok {
|
|
401
|
+
go func() {
|
|
402
|
+
select {
|
|
403
|
+
case fatalErr := <-fatalEngine.FatalErrors():
|
|
404
|
+
if fatalErr != nil {
|
|
405
|
+
slog.Error("durable executor worker failed", "error", fatalErr)
|
|
406
|
+
}
|
|
407
|
+
stopServe()
|
|
408
|
+
case <-stopCtx.Done():
|
|
409
|
+
}
|
|
410
|
+
}()
|
|
411
|
+
}
|
|
365
412
|
|
|
366
413
|
deps := &serveDeps{
|
|
367
414
|
wf: wfSvc,
|
|
@@ -402,6 +449,29 @@ func serveRoot(ctx context.Context, p paths.Paths, recover bool) error {
|
|
|
402
449
|
return serveResult
|
|
403
450
|
}
|
|
404
451
|
|
|
452
|
+
func recoveryBackupStem(database, stamp string, suffixes []string) (string, error) {
|
|
453
|
+
for attempt := 0; attempt < 1000; attempt++ {
|
|
454
|
+
candidateStamp := stamp
|
|
455
|
+
if attempt > 0 {
|
|
456
|
+
candidateStamp = fmt.Sprintf("%s-%d", stamp, attempt)
|
|
457
|
+
}
|
|
458
|
+
stem := fmt.Sprintf("%s.recover-%s", database, candidateStamp)
|
|
459
|
+
collision := false
|
|
460
|
+
for _, suffix := range suffixes {
|
|
461
|
+
if _, err := os.Stat(stem + ".bak" + suffix); err == nil {
|
|
462
|
+
collision = true
|
|
463
|
+
break
|
|
464
|
+
} else if !os.IsNotExist(err) {
|
|
465
|
+
return "", fmt.Errorf("inspect recovery backup %s: %w", stem+".bak"+suffix, err)
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
if !collision {
|
|
469
|
+
return stem, nil
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
return "", fmt.Errorf("unable to allocate unique recovery backup for %s", database)
|
|
473
|
+
}
|
|
474
|
+
|
|
405
475
|
func workflowConfigValidator(repoReg *repo.Registry) func(context.Context, *workflow.Workflow) error {
|
|
406
476
|
return func(ctx context.Context, wf *workflow.Workflow) error {
|
|
407
477
|
nodeCfgs := map[string]config.RawValues{}
|
|
@@ -431,10 +501,19 @@ func (r repoExists) Exists(name string) bool {
|
|
|
431
501
|
|
|
432
502
|
// serveDeps adapts composition-root services to server.Deps. Thin forwarder;
|
|
433
503
|
// no logic. Signatures match docs/structs-methods-interfaces.md Client.
|
|
504
|
+
type durableEngine interface {
|
|
505
|
+
runsvc.Executor
|
|
506
|
+
runsvc.RunQueries
|
|
507
|
+
Start(context.Context) error
|
|
508
|
+
Shutdown(context.Context) error
|
|
509
|
+
HasProcessedReport(context.Context, runsvc.ID, string) (bool, error)
|
|
510
|
+
RegisterNodeSession(context.Context, runsvc.NodeRuntimeRegistration) (runsvc.NodeRuntimeRegistrationAck, error)
|
|
511
|
+
}
|
|
512
|
+
|
|
434
513
|
type serveDeps struct {
|
|
435
514
|
wf *workflow.Service
|
|
436
515
|
repos *repo.Service
|
|
437
|
-
engine
|
|
516
|
+
engine durableEngine
|
|
438
517
|
runManager *runsvc.RunManager
|
|
439
518
|
onReposChanged func()
|
|
440
519
|
shutdown func(context.Context) error
|