relay-flow 0.2.2-alpha → 0.2.3-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -3
- package/cmd/relay-flow/commands_test.go +5 -1
- package/cmd/relay-flow/main.go +25 -1
- package/cmd/relay-flow/serve.go +12 -0
- package/internal/execution/goworkflows/activities.go +19 -0
- package/internal/execution/goworkflows/engine.go +20 -0
- package/internal/execution/goworkflows/engine_test.go +1 -1
- package/internal/execution/goworkflows/fakes_test.go +34 -6
- package/internal/execution/goworkflows/interpreter.go +48 -1
- package/internal/execution/goworkflows/projection.go +52 -11
- package/internal/execution/goworkflows/recovery_test.go +129 -0
- package/internal/harness/opencode/opencode.go +4 -4
- package/internal/harness/opencode/opencode_test.go +59 -4
- package/internal/harness/opencode/repo_setup.go +42 -2
- package/internal/identity/identity.go +28 -1
- package/internal/identity/identity_test.go +40 -0
- package/internal/run/manager.go +193 -25
- package/internal/run/run.go +24 -6
- package/internal/run/run_manager_test.go +100 -0
- package/internal/server/api_test.go +54 -0
- package/internal/server/client.go +11 -0
- package/internal/server/fixture_test.go +18 -0
- package/internal/server/server.go +16 -0
- package/internal/task/beads/beads.go +16 -0
- package/internal/task/beads/status_compatibility_test.go +43 -0
- package/internal/task/jira/helpers_test.go +3 -0
- package/internal/task/jira/jira.go +16 -0
- package/internal/task/jira/transition_defaults_test.go +43 -0
- package/internal/task/task.go +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -25,7 +25,7 @@ This is a ground-up rewrite. The previous per-workflow, in-memory daemon is gone
|
|
|
25
25
|
go install github.com/rajpopat27/relay-flow/cmd/relay-flow@latest
|
|
26
26
|
```
|
|
27
27
|
|
|
28
|
-
OpenCode plugin
|
|
28
|
+
OpenCode plugin configuration uses both entrypoints. The server entrypoint is listed in `opencode.json`:
|
|
29
29
|
|
|
30
30
|
```json
|
|
31
31
|
{
|
|
@@ -34,7 +34,22 @@ OpenCode plugin: add `"relay-flow-plugin"` to the `plugin` array in your repo's
|
|
|
34
34
|
}
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
-
The
|
|
37
|
+
The native HITL approval entrypoint is listed in `.opencode/tui.json`:
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{
|
|
41
|
+
"$schema": "https://opencode.ai/tui.json",
|
|
42
|
+
"plugin": ["relay-flow-plugin"]
|
|
43
|
+
}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The OpenCode harness adds both entries when a repo is registered. The server
|
|
47
|
+
entrypoint registers sessions, handles agent reports, and nudges invalid agent
|
|
48
|
+
output. The TUI entrypoint handles only HITL reports: after a valid completed
|
|
49
|
+
assistant report it shows a native Approve/Reject dialog. Approval delivers
|
|
50
|
+
`{runId, node, reportId, report}` via `relay-flow report` with retry; rejection
|
|
51
|
+
delivers nothing. `reportId` comes from the harness session/message identity;
|
|
52
|
+
`nodeVisitID` is internal and is never part of either plugin payload.
|
|
38
53
|
|
|
39
54
|
### One-time machine setup
|
|
40
55
|
|
|
@@ -309,7 +324,27 @@ EXPECTED RESULT: ...
|
|
|
309
324
|
|
|
310
325
|
The labels above are fixed; configurable templates do not change the parsed report contract. The plugin submits one `report` object containing both lower-camel `summary` and `feedback` objects. Relay-flow validates that complete shape once, renders `summaryReport` through the task system's summary-comment template on the current mailbox, and renders `feedbackReport` through its feedback-comment template on only the selected next mailbox. `None` is the literal marker for an intentionally empty section. When `NEXT STEP` is `end`, every FEEDBACK field must be `None` and no feedback comment is written.
|
|
311
326
|
|
|
312
|
-
The plugin delivers `{runId, node, reportId, report}` as one JSON object via `relay-flow report` stdin with the shared backoff (initial 2s, factor 2, jitter 0.2, max 5m) until acknowledged. It derives `reportId` from the harness session/message identity. Duplicate/stale reports are acked safely with no repeated graph effects. Invalid agent output is nudged; invalid HITL output stays silent.
|
|
327
|
+
The plugin delivers `{runId, node, reportId, report}` as one JSON object via `relay-flow report` stdin with the shared backoff (initial 2s, factor 2, jitter 0.2, max 5m) until acknowledged. It derives `reportId` from the harness session/message identity. Duplicate/stale reports are acked safely with no repeated graph effects. Invalid agent output is nudged; invalid or missing HITL output stays silent, while a valid HITL report opens the native TUI approval dialog. Relay-flow HITL approval does not use OpenCode's Question tool.
|
|
328
|
+
|
|
329
|
+
---
|
|
330
|
+
|
|
331
|
+
## Canceled run restart
|
|
332
|
+
|
|
333
|
+
Cancellation is permanent for the current execution. A canceled ticket is not
|
|
334
|
+
restarted by polling or by a ticket-status change. Start a fresh attempt
|
|
335
|
+
explicitly:
|
|
336
|
+
|
|
337
|
+
```sh
|
|
338
|
+
relay-flow run restart --ticket PAY-101
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
The new attempt starts at `start`, preserves the existing worktree/mailboxes/
|
|
342
|
+
comments/labels, and uses a numeric attempt ID (`2`, `3`, ...), with a fenced
|
|
343
|
+
execution ID such as `payments/basicFlow/PAY-101~attempt~2`. If a human has
|
|
344
|
+
moved the parent ticket to an incompatible status, `run get` shows `blocked`
|
|
345
|
+
with an instruction to move it to an allowed active start status; relay-flow
|
|
346
|
+
retries automatically and never overwrites the human-owned status. Done/Closed
|
|
347
|
+
tickets are not reopened automatically.
|
|
313
348
|
|
|
314
349
|
---
|
|
315
350
|
|
|
@@ -47,7 +47,7 @@ func TestCommandSurfaceExists(t *testing.T) {
|
|
|
47
47
|
{"workflow", "submit", "--file", "x.yaml"}, {"workflow", "remove", "--name", "x"},
|
|
48
48
|
{"workflow", "list"}, {"workflow", "get", "--name", "x"},
|
|
49
49
|
{"repo", "register"}, {"repo", "remove", "--name", "x"}, {"repo", "list"}, {"repo", "get", "--name", "x"},
|
|
50
|
-
{"run", "list"}, {"run", "get", "--ticket", "PAY-101"}, {"run", "cancel", "--ticket", "PAY-101"},
|
|
50
|
+
{"run", "list"}, {"run", "get", "--ticket", "PAY-101"}, {"run", "restart", "--ticket", "PAY-101"}, {"run", "cancel", "--ticket", "PAY-101"},
|
|
51
51
|
}
|
|
52
52
|
for _, argv := range commands {
|
|
53
53
|
// Recognized commands do not exit 2 ("usage/unknown"); they may exit
|
|
@@ -75,6 +75,7 @@ func TestRequiredFlagMissingExits2(t *testing.T) {
|
|
|
75
75
|
{"repo", "remove"}, // missing --name
|
|
76
76
|
{"repo", "get"}, // missing --name
|
|
77
77
|
{"run", "get"}, // missing --ticket
|
|
78
|
+
{"run", "restart"}, // missing --ticket
|
|
78
79
|
{"run", "cancel"}, // missing --ticket
|
|
79
80
|
} {
|
|
80
81
|
if code := cli(t, home, "", argv...); code != 2 {
|
|
@@ -1004,6 +1005,9 @@ func (s *ackServer) ListRuns(context.Context, runsvc.Filter) ([]runsvc.Run, erro
|
|
|
1004
1005
|
func (s *ackServer) GetRunByTicket(context.Context, string) (runsvc.Run, error) {
|
|
1005
1006
|
panic("unreachable")
|
|
1006
1007
|
}
|
|
1008
|
+
func (s *ackServer) RestartRun(context.Context, string) (runsvc.Run, error) {
|
|
1009
|
+
panic("unreachable")
|
|
1010
|
+
}
|
|
1007
1011
|
func (s *ackServer) CancelRun(context.Context, string, string) error { panic("unreachable") }
|
|
1008
1012
|
func (s *ackServer) DiscoverRepos(context.Context) ([]runner.RepoCandidate, error) {
|
|
1009
1013
|
panic("unreachable")
|
package/cmd/relay-flow/main.go
CHANGED
|
@@ -149,6 +149,7 @@ Usage:
|
|
|
149
149
|
|
|
150
150
|
relay-flow run list
|
|
151
151
|
relay-flow run get --ticket <key>
|
|
152
|
+
relay-flow run restart --ticket <key>
|
|
152
153
|
relay-flow run cancel --ticket <key>`)
|
|
153
154
|
}
|
|
154
155
|
|
|
@@ -946,6 +947,25 @@ func cmdRun(c *server.Client, args []string) int {
|
|
|
946
947
|
enc.SetIndent("", " ")
|
|
947
948
|
_ = enc.Encode(rn)
|
|
948
949
|
return exitOK
|
|
950
|
+
case "restart":
|
|
951
|
+
fs := flag.NewFlagSet("run restart", flag.ContinueOnError)
|
|
952
|
+
ticket := fs.String("ticket", "", "ticket key")
|
|
953
|
+
if err := fs.Parse(args[1:]); err != nil {
|
|
954
|
+
return exitUsage
|
|
955
|
+
}
|
|
956
|
+
if *ticket == "" {
|
|
957
|
+
fmt.Fprintln(os.Stderr, "run restart: --ticket is required")
|
|
958
|
+
return exitUsage
|
|
959
|
+
}
|
|
960
|
+
rn, err := c.RestartRun(context.Background(), *ticket)
|
|
961
|
+
if err != nil {
|
|
962
|
+
fmt.Fprintln(os.Stderr, err)
|
|
963
|
+
return exitFail
|
|
964
|
+
}
|
|
965
|
+
enc := json.NewEncoder(os.Stdout)
|
|
966
|
+
enc.SetIndent("", " ")
|
|
967
|
+
_ = enc.Encode(rn)
|
|
968
|
+
return exitOK
|
|
949
969
|
case "cancel":
|
|
950
970
|
fs := flag.NewFlagSet("run cancel", flag.ContinueOnError)
|
|
951
971
|
ticket := fs.String("ticket", "", "ticket key")
|
|
@@ -968,7 +988,11 @@ func cmdRun(c *server.Client, args []string) int {
|
|
|
968
988
|
}
|
|
969
989
|
|
|
970
990
|
func formatRunListRow(r runsvc.Run) string {
|
|
971
|
-
|
|
991
|
+
attempt := r.AttemptID
|
|
992
|
+
if attempt == 0 {
|
|
993
|
+
attempt = 1
|
|
994
|
+
}
|
|
995
|
+
row := fmt.Sprintf("%s\t%s\t%s\t%s\tattempt=%d", r.ID, r.Ticket.Key, r.Workflow, r.State, attempt)
|
|
972
996
|
if r.Retry != nil {
|
|
973
997
|
row += fmt.Sprintf("\tretrying attempt=%d next=%s error=%q",
|
|
974
998
|
r.Retry.Attempt, r.Retry.NextRetryAt.Format(time.RFC3339), r.Retry.LastError)
|
package/cmd/relay-flow/serve.go
CHANGED
|
@@ -249,6 +249,8 @@ func serveRoot(ctx context.Context, p paths.Paths, recover bool) error {
|
|
|
249
249
|
// the SAME registry by pointer (via replaceInternal), so the engine,
|
|
250
250
|
// pollers, and handlers observe one in-memory repo set.
|
|
251
251
|
wfSvc := workflow.NewService(store, engine, repoExists{repoReg})
|
|
252
|
+
runManager.Repos = repoReg
|
|
253
|
+
runManager.Workflows = wfSvc.Registry()
|
|
252
254
|
wfSvc.Gate = lifecycleGate
|
|
253
255
|
wfSvc.ValidateTaskConfig = workflowConfigValidator(repoReg)
|
|
254
256
|
// Submit/Remove must also rebuild repo bindings under the gate (spec
|
|
@@ -455,6 +457,16 @@ func (d *serveDeps) ListRuns(ctx context.Context, filter runsvc.Filter) ([]runsv
|
|
|
455
457
|
func (d *serveDeps) GetRunByTicket(ctx context.Context, ticket string) (runsvc.Run, error) {
|
|
456
458
|
return d.engine.FindRunByTicket(ctx, ticket)
|
|
457
459
|
}
|
|
460
|
+
func (d *serveDeps) RestartRun(ctx context.Context, ticket string) (runsvc.Run, error) {
|
|
461
|
+
rn, err := d.runManager.RestartByTicket(ctx, ticket)
|
|
462
|
+
if err != nil {
|
|
463
|
+
if errors.Is(err, runsvc.ErrRestartConflict) {
|
|
464
|
+
return runsvc.Run{}, fmt.Errorf("%w: %v", server.ErrConflict, err)
|
|
465
|
+
}
|
|
466
|
+
return runsvc.Run{}, err
|
|
467
|
+
}
|
|
468
|
+
return rn, nil
|
|
469
|
+
}
|
|
458
470
|
func (d *serveDeps) CancelRun(ctx context.Context, ticket, reason string) error {
|
|
459
471
|
return d.runManager.CancelByTicket(ctx, ticket, reason)
|
|
460
472
|
}
|
|
@@ -64,6 +64,25 @@ func (a *Activities) EnsureMailboxes(ctx context.Context, w run.Work, specs []ta
|
|
|
64
64
|
return sys.EnsureMailboxes(ctx, w.Parent, w.Workflow, specs)
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
// PrepareRestart reopens mailbox state through the optional task-system
|
|
68
|
+
// capability, then closes any surviving run-owned terminals while preserving
|
|
69
|
+
// the ticket worktree. Both operations are idempotent/retryable and remain
|
|
70
|
+
// behind their respective task and runner interfaces.
|
|
71
|
+
func (a *Activities) PrepareRestart(ctx context.Context, w run.Work, repoPath string, mailboxes []task.Mailbox) error {
|
|
72
|
+
sys, err := a.taskSystem(w.Repo)
|
|
73
|
+
if err != nil {
|
|
74
|
+
return err
|
|
75
|
+
}
|
|
76
|
+
if preparer, ok := sys.(task.RestartPreparer); ok {
|
|
77
|
+
if err := preparer.PrepareRestart(ctx, w.Parent, mailboxes); err != nil {
|
|
78
|
+
return err
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
spec := a.runSpec(w)
|
|
82
|
+
spec.RepoPath = repoPath
|
|
83
|
+
return a.Runner.CloseTerminals(ctx, spec)
|
|
84
|
+
}
|
|
85
|
+
|
|
67
86
|
// ValidateAgents validates every referenced agent on the repo.
|
|
68
87
|
func (a *Activities) ValidateAgents(ctx context.Context, repoPath string, agents []string) error {
|
|
69
88
|
for _, agent := range agents {
|
|
@@ -24,6 +24,7 @@ import (
|
|
|
24
24
|
"github.com/google/uuid"
|
|
25
25
|
|
|
26
26
|
"github.com/rajpopat27/relay-flow/internal/harness"
|
|
27
|
+
"github.com/rajpopat27/relay-flow/internal/identity"
|
|
27
28
|
"github.com/rajpopat27/relay-flow/internal/repo"
|
|
28
29
|
"github.com/rajpopat27/relay-flow/internal/run"
|
|
29
30
|
"github.com/rajpopat27/relay-flow/internal/runner"
|
|
@@ -220,6 +221,7 @@ func (e *Engine) registerActivities() error {
|
|
|
220
221
|
a := e.activities
|
|
221
222
|
for _, act := range []goworkflow.Activity{
|
|
222
223
|
a.EnsureMailboxes,
|
|
224
|
+
a.PrepareRestart,
|
|
223
225
|
a.ValidateAgents,
|
|
224
226
|
a.ApplyTaskConfig,
|
|
225
227
|
a.EnsureEnvironment,
|
|
@@ -283,6 +285,12 @@ func (e *Engine) Shutdown(ctx context.Context) error {
|
|
|
283
285
|
// title and sends the reconcile signal only when that terminal is missing
|
|
284
286
|
// or unusable. Repeated polls are harmless.
|
|
285
287
|
func (e *Engine) EnsureRun(ctx context.Context, start run.Start) (bool, error) {
|
|
288
|
+
if start.LogicalID == "" {
|
|
289
|
+
start.LogicalID = run.ID(identity.LogicalRunID(start.ID))
|
|
290
|
+
}
|
|
291
|
+
if start.AttemptID == 0 {
|
|
292
|
+
start.AttemptID = 1
|
|
293
|
+
}
|
|
286
294
|
r, err := e.runs.get(ctx, start.ID)
|
|
287
295
|
if errors.Is(err, errRunNotFound) {
|
|
288
296
|
start.Runtime = e.runtime
|
|
@@ -358,6 +366,18 @@ func (e *Engine) EnsureRun(ctx context.Context, start run.Start) (bool, error) {
|
|
|
358
366
|
// Attrs always carry ticket/runID/node/nodeVisitID when known.
|
|
359
367
|
func (e *Engine) SubmitReport(ctx context.Context, req run.ReportRequest) (run.ReportAck, error) {
|
|
360
368
|
r, err := e.runs.get(ctx, req.RunID)
|
|
369
|
+
if errors.Is(err, errRunNotFound) {
|
|
370
|
+
// A retained newer attempt can outlive an old attempt row. Resolve the
|
|
371
|
+
// stable logical ID and acknowledge the old attempt as a stale
|
|
372
|
+
// duplicate; it must never be validated or signaled into the new run.
|
|
373
|
+
logicalID := run.ID(identity.LogicalRunID(req.RunID))
|
|
374
|
+
if latest, lookupErr := e.runs.findByLogicalID(ctx, logicalID); lookupErr == nil && latest.ID != req.RunID {
|
|
375
|
+
slog.Info("report duplicate ack", "ticket", latest.Ticket.Key,
|
|
376
|
+
"runID", string(req.RunID), "logicalRunID", string(logicalID),
|
|
377
|
+
"node", req.Node, "reportID", req.ReportID, "state", string(latest.State))
|
|
378
|
+
return run.ReportAck{Accepted: true, Duplicate: true}, nil
|
|
379
|
+
}
|
|
380
|
+
}
|
|
361
381
|
if err != nil {
|
|
362
382
|
return run.ReportAck{}, fmt.Errorf("resolve run %s: %w", req.RunID, err)
|
|
363
383
|
}
|
|
@@ -71,7 +71,7 @@ func TestMailboxDescriptionAndLaunchPromptAreTaskSystemNeutral(t *testing.T) {
|
|
|
71
71
|
if err != nil {
|
|
72
72
|
t.Fatal(err)
|
|
73
73
|
}
|
|
74
|
-
want := "Task system: " + taskSystem + "\nUse the " + taskSystem + " tools to read the parent ticket PAY-101.\n\nYour mailbox is PAY-234. Read its description and comments for node instructions and feedback.\n\
|
|
74
|
+
want := "Task system: " + taskSystem + "\nUse the " + taskSystem + " tools to read the parent ticket PAY-101.\n\nYour mailbox is PAY-234. Read its description and comments for node instructions and feedback.\n\nReturn the complete report directly. Relay-flow will show a native TUI approval dialog after the report is valid. Do not use OpenCode's Question tool for relay-flow approval."
|
|
75
75
|
if prompt != want {
|
|
76
76
|
t.Fatalf("RenderPrompt(%s) = %q, want %q", taskSystem, prompt, want)
|
|
77
77
|
}
|
|
@@ -82,6 +82,7 @@ type fakeTaskSystem struct {
|
|
|
82
82
|
completeFail int // next N CompleteMailbox calls return transient error
|
|
83
83
|
completeConflict bool // CompleteMailbox returns retry.ConflictError
|
|
84
84
|
completeSlow time.Duration // CompleteMailbox sleeps (a running activity)
|
|
85
|
+
startConflict bool // start ApplyTaskConfig returns a status conflict
|
|
85
86
|
failComments bool // Comment returns transient error
|
|
86
87
|
|
|
87
88
|
// Recovery fixtures.
|
|
@@ -165,6 +166,15 @@ func (s *fakeTaskSystem) EnsureMailboxes(_ context.Context, parent task.TicketRe
|
|
|
165
166
|
|
|
166
167
|
func (s *fakeTaskSystem) ApplyTaskConfig(_ context.Context, target task.Target, cfg config.RawValues) error {
|
|
167
168
|
key := target.Parent.Key
|
|
169
|
+
if target.Mailbox == nil {
|
|
170
|
+
s.mu.Lock()
|
|
171
|
+
conflict := s.startConflict
|
|
172
|
+
s.mu.Unlock()
|
|
173
|
+
if conflict {
|
|
174
|
+
s.log.add("applyTaskConfigConflict:" + key)
|
|
175
|
+
return retry.ConflictError(errStartConflict)
|
|
176
|
+
}
|
|
177
|
+
}
|
|
168
178
|
if target.Mailbox != nil {
|
|
169
179
|
key = target.Mailbox.Key
|
|
170
180
|
s.mu.Lock()
|
|
@@ -227,6 +237,16 @@ func (s *fakeTaskSystem) Comment(_ context.Context, target task.Target, body, ma
|
|
|
227
237
|
return nil
|
|
228
238
|
}
|
|
229
239
|
|
|
240
|
+
func (s *fakeTaskSystem) PrepareRestart(_ context.Context, parent task.TicketRef, mbs []task.Mailbox) error {
|
|
241
|
+
s.mu.Lock()
|
|
242
|
+
for _, mb := range mbs {
|
|
243
|
+
s.mailboxStatus[mb.Key] = "To Do"
|
|
244
|
+
}
|
|
245
|
+
s.mu.Unlock()
|
|
246
|
+
s.log.add("prepareRestart:" + parent.Key)
|
|
247
|
+
return nil
|
|
248
|
+
}
|
|
249
|
+
|
|
230
250
|
func (s *fakeTaskSystem) ResetForRecovery(_ context.Context, parent task.TicketRef, mbs []task.Mailbox, _ config.RawValues) error {
|
|
231
251
|
s.mu.Lock()
|
|
232
252
|
for _, mb := range mbs {
|
|
@@ -287,6 +307,12 @@ func (s *fakeTaskSystem) setMailboxStatus(key, status string) {
|
|
|
287
307
|
s.mu.Unlock()
|
|
288
308
|
}
|
|
289
309
|
|
|
310
|
+
func (s *fakeTaskSystem) setStartConflict(value bool) {
|
|
311
|
+
s.mu.Lock()
|
|
312
|
+
s.startConflict = value
|
|
313
|
+
s.mu.Unlock()
|
|
314
|
+
}
|
|
315
|
+
|
|
290
316
|
// seedMailbox pre-populates an existing mailbox (with labels) for recovery
|
|
291
317
|
// and reuse tests.
|
|
292
318
|
func (s *fakeTaskSystem) seedMailbox(parentKey string, mb task.Mailbox, labels []string) {
|
|
@@ -345,11 +371,12 @@ func (f *fakeRunner) ValidateRepo(context.Context, string, string) error { retur
|
|
|
345
371
|
func (f *fakeRunner) EnsureEnvironment(_ context.Context, spec runner.RunSpec) (runner.Environment, error) {
|
|
346
372
|
f.mu.Lock()
|
|
347
373
|
defer f.mu.Unlock()
|
|
348
|
-
|
|
374
|
+
key := spec.TicketKey
|
|
375
|
+
if e, ok := f.envs[key]; ok {
|
|
349
376
|
return e, nil
|
|
350
377
|
}
|
|
351
|
-
e := runner.Environment{ID: "env-" +
|
|
352
|
-
f.envs[
|
|
378
|
+
e := runner.Environment{ID: "env-" + spec.TicketKey, Path: spec.RepoPath}
|
|
379
|
+
f.envs[key] = e
|
|
353
380
|
f.log.add("ensureEnvironment:" + string(spec.RunID))
|
|
354
381
|
return e, nil
|
|
355
382
|
}
|
|
@@ -418,7 +445,7 @@ func (f *fakeRunner) CloseTerminal(_ context.Context, t runner.Terminal) error {
|
|
|
418
445
|
func (f *fakeRunner) CloseTerminals(_ context.Context, spec runner.RunSpec) error {
|
|
419
446
|
f.mu.Lock()
|
|
420
447
|
defer f.mu.Unlock()
|
|
421
|
-
prefix := "env-" +
|
|
448
|
+
prefix := "env-" + spec.TicketKey + "/"
|
|
422
449
|
for k, ft := range f.terminals {
|
|
423
450
|
if len(k) >= len(prefix) && k[:len(prefix)] == prefix {
|
|
424
451
|
ft.live = false
|
|
@@ -433,13 +460,13 @@ func (f *fakeRunner) CloseTerminals(_ context.Context, spec runner.RunSpec) erro
|
|
|
433
460
|
func (f *fakeRunner) CleanupRun(_ context.Context, spec runner.RunSpec) error {
|
|
434
461
|
f.mu.Lock()
|
|
435
462
|
defer f.mu.Unlock()
|
|
436
|
-
prefix := "env-" +
|
|
463
|
+
prefix := "env-" + spec.TicketKey + "/"
|
|
437
464
|
for k := range f.terminals {
|
|
438
465
|
if len(k) >= len(prefix) && k[:len(prefix)] == prefix {
|
|
439
466
|
delete(f.terminals, k)
|
|
440
467
|
}
|
|
441
468
|
}
|
|
442
|
-
delete(f.envs,
|
|
469
|
+
delete(f.envs, spec.TicketKey)
|
|
443
470
|
f.cleaned = append(f.cleaned, string(spec.RunID))
|
|
444
471
|
f.log.add("cleanupRun:" + string(spec.RunID))
|
|
445
472
|
return nil
|
|
@@ -553,3 +580,4 @@ type conflictError struct{ msg string }
|
|
|
553
580
|
func (e *conflictError) Error() string { return e.msg }
|
|
554
581
|
|
|
555
582
|
var errConflict = &conflictError{msg: "human moved mailbox"}
|
|
583
|
+
var errStartConflict = &conflictError{msg: "human moved ticket status"}
|
|
@@ -57,10 +57,18 @@ func jitter(ctx goworkflow.Context) (float64, error) {
|
|
|
57
57
|
// deterministic. On cancellation it runs cancellation cleanup on a
|
|
58
58
|
// disconnected context.
|
|
59
59
|
func (a *Activities) TicketWorkflow(ctx goworkflow.Context, start run.Start) error {
|
|
60
|
+
if start.LogicalID == "" {
|
|
61
|
+
start.LogicalID = run.ID(identity.LogicalRunID(start.ID))
|
|
62
|
+
}
|
|
63
|
+
if start.AttemptID == 0 {
|
|
64
|
+
start.AttemptID = 1
|
|
65
|
+
}
|
|
60
66
|
err := a.runGraph(ctx, start)
|
|
61
67
|
if err != nil && ctx.Err() != nil {
|
|
62
68
|
work := run.Work{
|
|
63
69
|
RunID: start.ID,
|
|
70
|
+
LogicalID: start.LogicalID,
|
|
71
|
+
AttemptID: start.AttemptID,
|
|
64
72
|
Repo: start.Repo,
|
|
65
73
|
Workflow: start.Workflow.Name,
|
|
66
74
|
Parent: start.Ticket,
|
|
@@ -76,6 +84,8 @@ func (a *Activities) runGraph(ctx goworkflow.Context, start run.Start) error {
|
|
|
76
84
|
wf := start.Workflow // value snapshot
|
|
77
85
|
work := run.Work{
|
|
78
86
|
RunID: start.ID,
|
|
87
|
+
LogicalID: start.LogicalID,
|
|
88
|
+
AttemptID: start.AttemptID,
|
|
79
89
|
Repo: start.Repo,
|
|
80
90
|
Workflow: wf.Name,
|
|
81
91
|
Parent: start.Ticket,
|
|
@@ -93,6 +103,26 @@ func (a *Activities) runGraph(ctx goworkflow.Context, start run.Start) error {
|
|
|
93
103
|
return err
|
|
94
104
|
}
|
|
95
105
|
|
|
106
|
+
// An explicit restart reuses the task-system mailboxes and ticket
|
|
107
|
+
// worktree, but resets relay-owned mailbox state and closes stale node
|
|
108
|
+
// terminals before the fresh start edge is processed. Human-owned
|
|
109
|
+
// incompatible states are returned as conflicts and keep this attempt
|
|
110
|
+
// blocked until the human restores a compatible state.
|
|
111
|
+
if start.AttemptID > 1 {
|
|
112
|
+
mailboxList := make([]task.Mailbox, 0, len(mailboxes))
|
|
113
|
+
for _, mailbox := range mailboxes {
|
|
114
|
+
mailboxList = append(mailboxList, mailbox)
|
|
115
|
+
}
|
|
116
|
+
sort.Slice(mailboxList, func(i, j int) bool { return mailboxList[i].Node < mailboxList[j].Node })
|
|
117
|
+
if _, err := retryLoop(ctx, start.ID, a, work, "start",
|
|
118
|
+
func(ctx2 goworkflow.Context) goworkflow.Future[struct{}] {
|
|
119
|
+
return goworkflow.ExecuteActivity[struct{}](ctx2, noNativeRetries,
|
|
120
|
+
a.PrepareRestart, work, start.RepoPath, mailboxList)
|
|
121
|
+
}); err != nil {
|
|
122
|
+
return err
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
96
126
|
// Validate every referenced agent before the start edge.
|
|
97
127
|
agentSet := map[string]bool{}
|
|
98
128
|
for _, n := range wf.Nodes {
|
|
@@ -443,6 +473,7 @@ func retryLoop[T any](ctx goworkflow.Context, id run.ID, a *Activities, work run
|
|
|
443
473
|
}
|
|
444
474
|
f := classifyActivityError(err)
|
|
445
475
|
if f.Kind == retry.Conflict {
|
|
476
|
+
f.Message = blockedMessage(work, node, f.Message)
|
|
446
477
|
// Mark blocked, keep retrying on the capped schedule.
|
|
447
478
|
_, _ = scheduleState(ctx, a, id, run.StateBlocked, f.Message)
|
|
448
479
|
blocked = true
|
|
@@ -463,6 +494,18 @@ func retryLoop[T any](ctx goworkflow.Context, id run.ID, a *Activities, work run
|
|
|
463
494
|
}
|
|
464
495
|
}
|
|
465
496
|
|
|
497
|
+
func blockedMessage(work run.Work, node, message string) string {
|
|
498
|
+
message = strings.TrimRight(message, ". ")
|
|
499
|
+
lower := strings.ToLower(message)
|
|
500
|
+
if node == "start" && !strings.Contains(lower, "mailbox") {
|
|
501
|
+
return fmt.Sprintf("%s. Move ticket %s to an allowed active start status; relay-flow will retry automatically", message, work.Parent.Key)
|
|
502
|
+
}
|
|
503
|
+
if node != "" {
|
|
504
|
+
return fmt.Sprintf("%s. Restore the task-system state required for node %s; relay-flow will retry automatically", message, node)
|
|
505
|
+
}
|
|
506
|
+
return fmt.Sprintf("%s. Restore the task-system state required by this operation; relay-flow will retry automatically", message)
|
|
507
|
+
}
|
|
508
|
+
|
|
466
509
|
// logRetry emits the 9.6 retry-classification info line, replay-safe.
|
|
467
510
|
// Attrs come from the always-known run.Work value carried by the caller,
|
|
468
511
|
// so ticket/repo/workflow are present even if the projection is briefly
|
|
@@ -565,6 +608,10 @@ func mustJitter(ctx goworkflow.Context) float64 {
|
|
|
565
608
|
// canceled. No rollback/compensation ever runs.
|
|
566
609
|
func (a *Activities) cancelCleanup(ctx goworkflow.Context, work run.Work, repoPath, reason string) error {
|
|
567
610
|
dctx := goworkflow.NewDisconnectedContext(ctx)
|
|
611
|
+
markerID := work.LogicalID
|
|
612
|
+
if markerID == "" {
|
|
613
|
+
markerID = work.RunID
|
|
614
|
+
}
|
|
568
615
|
if _, err := retryLoop(dctx, work.RunID, a, work, "",
|
|
569
616
|
func(ctx2 goworkflow.Context) goworkflow.Future[struct{}] {
|
|
570
617
|
return goworkflow.ExecuteActivity[struct{}](ctx2, noNativeRetries,
|
|
@@ -578,7 +625,7 @@ func (a *Activities) cancelCleanup(ctx goworkflow.Context, work run.Work, repoPa
|
|
|
578
625
|
RunID: work.RunID,
|
|
579
626
|
Item: task.Target{Parent: work.Parent},
|
|
580
627
|
Body: "Run canceled: " + reason,
|
|
581
|
-
Marker: run.CancellationMarker(
|
|
628
|
+
Marker: run.CancellationMarker(markerID),
|
|
582
629
|
})
|
|
583
630
|
}); err != nil {
|
|
584
631
|
return err
|
|
@@ -34,6 +34,8 @@ type NodeRuntime struct {
|
|
|
34
34
|
const relayRunsSchema = `
|
|
35
35
|
CREATE TABLE IF NOT EXISTS relay_runs (
|
|
36
36
|
id TEXT PRIMARY KEY,
|
|
37
|
+
logical_run_id TEXT,
|
|
38
|
+
attempt_id INTEGER,
|
|
37
39
|
repo TEXT NOT NULL,
|
|
38
40
|
workflow TEXT NOT NULL,
|
|
39
41
|
ticket_id TEXT NOT NULL,
|
|
@@ -86,9 +88,11 @@ func (p *RunProjection) migrate() error {
|
|
|
86
88
|
return err
|
|
87
89
|
}
|
|
88
90
|
for name, definition := range map[string]string{
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
91
|
+
"logical_run_id": "TEXT",
|
|
92
|
+
"attempt_id": "INTEGER",
|
|
93
|
+
"retry_error": "TEXT",
|
|
94
|
+
"retry_attempt": "INTEGER",
|
|
95
|
+
"next_retry_at": "DATETIME",
|
|
92
96
|
} {
|
|
93
97
|
var count int
|
|
94
98
|
if err := p.DB.QueryRow(`SELECT COUNT(1) FROM pragma_table_info('relay_runs') WHERE name = ?`, name).Scan(&count); err != nil {
|
|
@@ -100,6 +104,15 @@ func (p *RunProjection) migrate() error {
|
|
|
100
104
|
}
|
|
101
105
|
}
|
|
102
106
|
}
|
|
107
|
+
// Rows created before attempt identities were introduced represent the
|
|
108
|
+
// original attempt. Backfill the stable logical ID and attempt number so
|
|
109
|
+
// restart allocation remains numeric and never reuses attempt 1.
|
|
110
|
+
if _, err := p.DB.Exec(`UPDATE relay_runs SET logical_run_id = id WHERE COALESCE(logical_run_id, '') = ''`); err != nil {
|
|
111
|
+
return err
|
|
112
|
+
}
|
|
113
|
+
if _, err := p.DB.Exec(`UPDATE relay_runs SET attempt_id = 1 WHERE attempt_id IS NULL OR attempt_id = 0`); err != nil {
|
|
114
|
+
return err
|
|
115
|
+
}
|
|
103
116
|
return nil
|
|
104
117
|
}
|
|
105
118
|
|
|
@@ -110,11 +123,19 @@ var errNodeRuntimeNotFound = errors.New("node runtime not found")
|
|
|
110
123
|
func IsNotFound(err error) bool { return errors.Is(err, errRunNotFound) }
|
|
111
124
|
|
|
112
125
|
func (p *RunProjection) insertStart(ctx context.Context, s run.Start, now time.Time) error {
|
|
126
|
+
logicalID := s.LogicalID
|
|
127
|
+
if logicalID == "" {
|
|
128
|
+
logicalID = s.ID
|
|
129
|
+
}
|
|
130
|
+
attemptID := s.AttemptID
|
|
131
|
+
if attemptID == 0 {
|
|
132
|
+
attemptID = 1
|
|
133
|
+
}
|
|
113
134
|
_, err := p.DB.ExecContext(ctx, `
|
|
114
|
-
INSERT INTO relay_runs (id, repo, workflow, ticket_id, ticket_key, state, started_at, updated_at)
|
|
115
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
135
|
+
INSERT INTO relay_runs (id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, started_at, updated_at)
|
|
136
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
116
137
|
ON CONFLICT(id) DO NOTHING`,
|
|
117
|
-
string(s.ID), s.Repo, s.Workflow.Name, s.Ticket.ID, s.Ticket.Key,
|
|
138
|
+
string(s.ID), string(logicalID), int64(attemptID), s.Repo, s.Workflow.Name, s.Ticket.ID, s.Ticket.Key,
|
|
118
139
|
string(run.StateStarting), now, now)
|
|
119
140
|
return err
|
|
120
141
|
}
|
|
@@ -360,7 +381,7 @@ func nullableString(value string) any {
|
|
|
360
381
|
|
|
361
382
|
func (p *RunProjection) get(ctx context.Context, id run.ID) (run.Run, error) {
|
|
362
383
|
row := p.DB.QueryRowContext(ctx, `
|
|
363
|
-
SELECT id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
384
|
+
SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
364
385
|
retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at
|
|
365
386
|
FROM relay_runs WHERE id = ?`, string(id))
|
|
366
387
|
return scanRun(row)
|
|
@@ -368,9 +389,17 @@ func (p *RunProjection) get(ctx context.Context, id run.ID) (run.Run, error) {
|
|
|
368
389
|
|
|
369
390
|
func (p *RunProjection) findByTicket(ctx context.Context, ticket string) (run.Run, error) {
|
|
370
391
|
row := p.DB.QueryRowContext(ctx, `
|
|
371
|
-
SELECT id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
392
|
+
SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
372
393
|
retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at
|
|
373
|
-
FROM relay_runs WHERE ticket_key = ? ORDER BY started_at DESC LIMIT 1`, ticket)
|
|
394
|
+
FROM relay_runs WHERE ticket_key = ? ORDER BY started_at DESC, attempt_id DESC LIMIT 1`, ticket)
|
|
395
|
+
return scanRun(row)
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
func (p *RunProjection) findByLogicalID(ctx context.Context, logicalID run.ID) (run.Run, error) {
|
|
399
|
+
row := p.DB.QueryRowContext(ctx, `
|
|
400
|
+
SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
401
|
+
retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at
|
|
402
|
+
FROM relay_runs WHERE logical_run_id = ? ORDER BY started_at DESC, attempt_id DESC LIMIT 1`, string(logicalID))
|
|
374
403
|
return scanRun(row)
|
|
375
404
|
}
|
|
376
405
|
|
|
@@ -380,11 +409,13 @@ type rowScanner interface {
|
|
|
380
409
|
|
|
381
410
|
func scanRun(row rowScanner) (run.Run, error) {
|
|
382
411
|
var r run.Run
|
|
412
|
+
var logicalID sql.NullString
|
|
413
|
+
var attemptNumber sql.NullInt64
|
|
383
414
|
var node, visit, lastErr, retryErr sql.NullString
|
|
384
415
|
var retryAttempt sql.NullInt64
|
|
385
416
|
var nextRetry, finished sql.NullTime
|
|
386
417
|
var started, updated time.Time
|
|
387
|
-
err := row.Scan(&r.ID, &r.Repo, &r.Workflow, &r.Ticket.ID, &r.Ticket.Key, &r.State,
|
|
418
|
+
err := row.Scan(&r.ID, &logicalID, &attemptNumber, &r.Repo, &r.Workflow, &r.Ticket.ID, &r.Ticket.Key, &r.State,
|
|
388
419
|
&node, &visit, &lastErr, &retryErr, &retryAttempt, &nextRetry, &started, &updated, &finished)
|
|
389
420
|
if errors.Is(err, sql.ErrNoRows) {
|
|
390
421
|
return run.Run{}, errRunNotFound
|
|
@@ -392,6 +423,16 @@ func scanRun(row rowScanner) (run.Run, error) {
|
|
|
392
423
|
if err != nil {
|
|
393
424
|
return run.Run{}, err
|
|
394
425
|
}
|
|
426
|
+
if logicalID.Valid && logicalID.String != "" {
|
|
427
|
+
r.LogicalID = run.ID(logicalID.String)
|
|
428
|
+
} else {
|
|
429
|
+
r.LogicalID = r.ID
|
|
430
|
+
}
|
|
431
|
+
if attemptNumber.Valid && attemptNumber.Int64 > 0 {
|
|
432
|
+
r.AttemptID = run.AttemptID(attemptNumber.Int64)
|
|
433
|
+
} else {
|
|
434
|
+
r.AttemptID = 1
|
|
435
|
+
}
|
|
395
436
|
r.CurrentNode = node.String
|
|
396
437
|
r.CurrentNodeVisitID = run.NodeVisitID(visit.String)
|
|
397
438
|
r.LastError = lastErr.String
|
|
@@ -410,7 +451,7 @@ func scanRun(row rowScanner) (run.Run, error) {
|
|
|
410
451
|
}
|
|
411
452
|
|
|
412
453
|
func (p *RunProjection) list(ctx context.Context, f run.Filter) ([]run.Run, error) {
|
|
413
|
-
q := `SELECT id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error, retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at FROM relay_runs WHERE 1=1`
|
|
454
|
+
q := `SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error, retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at FROM relay_runs WHERE 1=1`
|
|
414
455
|
var args []any
|
|
415
456
|
if f.Repo != "" {
|
|
416
457
|
q += ` AND repo = ?`
|