relay-flow 0.2.2-alpha → 0.2.4-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -13
- package/cmd/relay-flow/commands_test.go +5 -1
- package/cmd/relay-flow/main.go +27 -1
- package/cmd/relay-flow/pi_wiring_test.go +64 -0
- package/cmd/relay-flow/serve.go +14 -0
- package/examples/beads-workflow.yaml +3 -3
- package/examples/config-reference.yaml +144 -0
- package/examples/minimal-beads-task-workflow.yaml +34 -0
- package/examples/minimal-jira-task-workflow.yaml +68 -0
- package/examples/workflow-reference.yaml +111 -0
- package/internal/execution/goworkflows/activities.go +19 -0
- package/internal/execution/goworkflows/engine.go +20 -0
- package/internal/execution/goworkflows/engine_test.go +1 -1
- package/internal/execution/goworkflows/fakes_test.go +34 -6
- package/internal/execution/goworkflows/interpreter.go +48 -1
- package/internal/execution/goworkflows/projection.go +52 -11
- package/internal/execution/goworkflows/recovery_test.go +129 -0
- package/internal/harness/opencode/opencode.go +4 -4
- package/internal/harness/opencode/opencode_test.go +59 -4
- package/internal/harness/opencode/repo_setup.go +42 -2
- package/internal/harness/pi/config_test.go +46 -0
- package/internal/harness/pi/lifecycle_test.go +69 -0
- package/internal/harness/pi/pi.go +261 -0
- package/internal/harness/pi/pi_test.go +322 -0
- package/internal/harness/pi/prompt_test.go +121 -0
- package/internal/harness/pi/testdata/pi-0.84.1/capture.json +126 -0
- package/internal/harness/pi/testdata/pi-0.84.1/noninteractive-output.txt +11 -0
- package/internal/harness/pi/testdata/pi-0.84.1/tui-output-sanitized.txt +17 -0
- package/internal/harness/pi/validation_test.go +168 -0
- package/internal/identity/identity.go +28 -1
- package/internal/identity/identity_test.go +40 -0
- package/internal/run/manager.go +193 -25
- package/internal/run/run.go +24 -6
- package/internal/run/run_manager_test.go +100 -0
- package/internal/runner/herdr/baseref.go +60 -0
- package/internal/runner/herdr/herdr.go +578 -0
- package/internal/runner/herdr/herdr_test.go +688 -0
- package/internal/runner/herdr/herdrcli/contract.go +128 -0
- package/internal/runner/herdr/herdrcli/exec.go +68 -0
- package/internal/runner/herdr/herdrcli/herdrcli_test.go +274 -0
- package/internal/runner/herdr/herdrcli/live_test.go +132 -0
- package/internal/runner/herdr/herdrcli/operations.go +281 -0
- package/internal/runner/herdr/herdrcli/response.go +116 -0
- package/internal/runner/herdr/herdrcli/testdata/empty-panes.json +1 -0
- package/internal/runner/herdr/herdrcli/testdata/empty-tabs.json +1 -0
- package/internal/runner/herdr/herdrcli/testdata/error-not-git-worktree.json +1 -0
- package/internal/runner/herdr/herdrcli/testdata/error-pane-not-found.json +1 -0
- package/internal/runner/herdr/herdrcli/testdata/error-workspace-not-found.json +1 -0
- package/internal/runner/herdr/herdrcli/testdata/error-worktree-not-found.json +1 -0
- package/internal/runner/herdr/herdrcli/testdata/malformed.json +1 -0
- package/internal/runner/herdr/herdrcli/testdata/pane-close.json +6 -0
- package/internal/runner/herdr/herdrcli/testdata/pane-get.json +25 -0
- package/internal/runner/herdr/herdrcli/testdata/pane-list.json +45 -0
- package/internal/runner/herdr/herdrcli/testdata/pane-process-info-shell.json +22 -0
- package/internal/runner/herdr/herdrcli/testdata/pane-process-info.json +23 -0
- package/internal/runner/herdr/herdrcli/testdata/pane-rename.json +23 -0
- package/internal/runner/herdr/herdrcli/testdata/snapshot.json +213 -0
- package/internal/runner/herdr/herdrcli/testdata/strict-herdr.sh +175 -0
- package/internal/runner/herdr/herdrcli/testdata/tab-create.json +31 -0
- package/internal/runner/herdr/herdrcli/testdata/tab-list.json +26 -0
- package/internal/runner/herdr/herdrcli/testdata/workspace-close.json +6 -0
- package/internal/runner/herdr/herdrcli/testdata/worktree-create.json +58 -0
- package/internal/runner/herdr/herdrcli/testdata/worktree-list.json +35 -0
- package/internal/runner/herdr/herdrcli/testdata/worktree-open.json +59 -0
- package/internal/server/api_test.go +54 -0
- package/internal/server/client.go +11 -0
- package/internal/server/fixture_test.go +18 -0
- package/internal/server/server.go +16 -0
- package/internal/task/beads/beads.go +16 -16
- package/internal/task/beads/config_compatibility_test.go +7 -8
- package/internal/task/beads/status_compatibility_test.go +43 -0
- package/internal/task/jira/filters_test.go +32 -6
- package/internal/task/jira/helpers_test.go +3 -0
- package/internal/task/jira/jira.go +43 -17
- package/internal/task/jira/transition_defaults_test.go +43 -0
- package/internal/task/task.go +8 -0
- package/package.json +1 -1
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Complete workflow definition reference.
|
|
2
|
+
#
|
|
3
|
+
# This example uses Jira-native values and the OpenCode agent names build and
|
|
4
|
+
# plan. For Beads, use provider-native values: open, task, in_progress, and
|
|
5
|
+
# closed. For Pi, use default or a repository role from .pi/roles/<agent>.md.
|
|
6
|
+
|
|
7
|
+
name: referenceFlow
|
|
8
|
+
|
|
9
|
+
repos:
|
|
10
|
+
- payments
|
|
11
|
+
|
|
12
|
+
cleanupRunnerOnEnd: true
|
|
13
|
+
|
|
14
|
+
taskConfig:
|
|
15
|
+
# Optional assignee inherited by node mailboxes. It does not filter parent
|
|
16
|
+
# tickets; use filters.assignees for ticket pickup.
|
|
17
|
+
assignee: developer@example.com
|
|
18
|
+
|
|
19
|
+
filters:
|
|
20
|
+
parentStatuses:
|
|
21
|
+
- To Do
|
|
22
|
+
issueTypes:
|
|
23
|
+
- Task
|
|
24
|
+
labels:
|
|
25
|
+
- relay-ready
|
|
26
|
+
assignees:
|
|
27
|
+
- currentUser()
|
|
28
|
+
# Jira-only symbolic value: resolves to the authenticated Jira email.
|
|
29
|
+
# Other assignee values are normalized Jira emails, not JQL expressions.
|
|
30
|
+
|
|
31
|
+
# filters.assignees controls ticket pickup independently of assignee.
|
|
32
|
+
# Use an explicit empty list to include all assignees while retaining any
|
|
33
|
+
# inherited mailbox assignment:
|
|
34
|
+
# filters:
|
|
35
|
+
# assignees: []
|
|
36
|
+
# To disable mailbox assignment too, set assignee: "" at this scope.
|
|
37
|
+
|
|
38
|
+
# Prefer lifecycle-specific transitionTo values on start, work nodes, and
|
|
39
|
+
# end. A value here is inherited by every lifecycle point that reads it.
|
|
40
|
+
# transitionTo:
|
|
41
|
+
# parentStatus: In Progress
|
|
42
|
+
# taskStatus: In Progress
|
|
43
|
+
|
|
44
|
+
nodes:
|
|
45
|
+
start:
|
|
46
|
+
# start has no type, agent, description, nudgePrompt, or failure routes.
|
|
47
|
+
taskConfig:
|
|
48
|
+
transitionTo:
|
|
49
|
+
parentStatus: In Progress
|
|
50
|
+
onSuccess:
|
|
51
|
+
- target: implement
|
|
52
|
+
when: The parent ticket is ready for implementation.
|
|
53
|
+
|
|
54
|
+
implement:
|
|
55
|
+
type: agent
|
|
56
|
+
agent: build
|
|
57
|
+
description: |
|
|
58
|
+
Implement the parent ticket in the ticket worktree.
|
|
59
|
+
Run the relevant tests and verification commands.
|
|
60
|
+
nudgePrompt: |
|
|
61
|
+
Continue working on {{ticket}} at node {{node}}.
|
|
62
|
+
Read the latest mailbox feedback.
|
|
63
|
+
Valid next steps are: {{nextSteps}}.
|
|
64
|
+
Return the complete report contract.
|
|
65
|
+
taskConfig:
|
|
66
|
+
transitionTo:
|
|
67
|
+
taskStatus: In Progress
|
|
68
|
+
onSuccess:
|
|
69
|
+
- target: review
|
|
70
|
+
when: Implementation and verification are complete.
|
|
71
|
+
onFailure:
|
|
72
|
+
- target: implement
|
|
73
|
+
when: Implementation needs another pass.
|
|
74
|
+
|
|
75
|
+
review:
|
|
76
|
+
type: agent
|
|
77
|
+
agent: plan
|
|
78
|
+
description: |
|
|
79
|
+
Review the implementation for correctness, regressions, missing tests,
|
|
80
|
+
and unresolved issues. Prepare the work for human PR review.
|
|
81
|
+
taskConfig:
|
|
82
|
+
transitionTo:
|
|
83
|
+
taskStatus: In Progress
|
|
84
|
+
onSuccess:
|
|
85
|
+
- target: prReview
|
|
86
|
+
when: The implementation is ready for human PR review.
|
|
87
|
+
onFailure:
|
|
88
|
+
- target: implement
|
|
89
|
+
when: The review found issues requiring implementation changes.
|
|
90
|
+
|
|
91
|
+
prReview:
|
|
92
|
+
type: hitl
|
|
93
|
+
agent: plan
|
|
94
|
+
description: |
|
|
95
|
+
Review the implementation and proposed changes with a human.
|
|
96
|
+
Approve the work or request implementation changes.
|
|
97
|
+
taskConfig:
|
|
98
|
+
transitionTo:
|
|
99
|
+
taskStatus: In Progress
|
|
100
|
+
onSuccess:
|
|
101
|
+
- target: end
|
|
102
|
+
when: The human approves the PR review.
|
|
103
|
+
onFailure:
|
|
104
|
+
- target: implement
|
|
105
|
+
when: The human requests changes.
|
|
106
|
+
|
|
107
|
+
end:
|
|
108
|
+
# end has no type, agent, description, nudgePrompt, or routes.
|
|
109
|
+
taskConfig:
|
|
110
|
+
transitionTo:
|
|
111
|
+
parentStatus: Done
|
|
@@ -64,6 +64,25 @@ func (a *Activities) EnsureMailboxes(ctx context.Context, w run.Work, specs []ta
|
|
|
64
64
|
return sys.EnsureMailboxes(ctx, w.Parent, w.Workflow, specs)
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
// PrepareRestart reopens mailbox state through the optional task-system
|
|
68
|
+
// capability, then closes any surviving run-owned terminals while preserving
|
|
69
|
+
// the ticket worktree. Both operations are idempotent/retryable and remain
|
|
70
|
+
// behind their respective task and runner interfaces.
|
|
71
|
+
func (a *Activities) PrepareRestart(ctx context.Context, w run.Work, repoPath string, mailboxes []task.Mailbox) error {
|
|
72
|
+
sys, err := a.taskSystem(w.Repo)
|
|
73
|
+
if err != nil {
|
|
74
|
+
return err
|
|
75
|
+
}
|
|
76
|
+
if preparer, ok := sys.(task.RestartPreparer); ok {
|
|
77
|
+
if err := preparer.PrepareRestart(ctx, w.Parent, mailboxes); err != nil {
|
|
78
|
+
return err
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
spec := a.runSpec(w)
|
|
82
|
+
spec.RepoPath = repoPath
|
|
83
|
+
return a.Runner.CloseTerminals(ctx, spec)
|
|
84
|
+
}
|
|
85
|
+
|
|
67
86
|
// ValidateAgents validates every referenced agent on the repo.
|
|
68
87
|
func (a *Activities) ValidateAgents(ctx context.Context, repoPath string, agents []string) error {
|
|
69
88
|
for _, agent := range agents {
|
|
@@ -24,6 +24,7 @@ import (
|
|
|
24
24
|
"github.com/google/uuid"
|
|
25
25
|
|
|
26
26
|
"github.com/rajpopat27/relay-flow/internal/harness"
|
|
27
|
+
"github.com/rajpopat27/relay-flow/internal/identity"
|
|
27
28
|
"github.com/rajpopat27/relay-flow/internal/repo"
|
|
28
29
|
"github.com/rajpopat27/relay-flow/internal/run"
|
|
29
30
|
"github.com/rajpopat27/relay-flow/internal/runner"
|
|
@@ -220,6 +221,7 @@ func (e *Engine) registerActivities() error {
|
|
|
220
221
|
a := e.activities
|
|
221
222
|
for _, act := range []goworkflow.Activity{
|
|
222
223
|
a.EnsureMailboxes,
|
|
224
|
+
a.PrepareRestart,
|
|
223
225
|
a.ValidateAgents,
|
|
224
226
|
a.ApplyTaskConfig,
|
|
225
227
|
a.EnsureEnvironment,
|
|
@@ -283,6 +285,12 @@ func (e *Engine) Shutdown(ctx context.Context) error {
|
|
|
283
285
|
// title and sends the reconcile signal only when that terminal is missing
|
|
284
286
|
// or unusable. Repeated polls are harmless.
|
|
285
287
|
func (e *Engine) EnsureRun(ctx context.Context, start run.Start) (bool, error) {
|
|
288
|
+
if start.LogicalID == "" {
|
|
289
|
+
start.LogicalID = run.ID(identity.LogicalRunID(start.ID))
|
|
290
|
+
}
|
|
291
|
+
if start.AttemptID == 0 {
|
|
292
|
+
start.AttemptID = 1
|
|
293
|
+
}
|
|
286
294
|
r, err := e.runs.get(ctx, start.ID)
|
|
287
295
|
if errors.Is(err, errRunNotFound) {
|
|
288
296
|
start.Runtime = e.runtime
|
|
@@ -358,6 +366,18 @@ func (e *Engine) EnsureRun(ctx context.Context, start run.Start) (bool, error) {
|
|
|
358
366
|
// Attrs always carry ticket/runID/node/nodeVisitID when known.
|
|
359
367
|
func (e *Engine) SubmitReport(ctx context.Context, req run.ReportRequest) (run.ReportAck, error) {
|
|
360
368
|
r, err := e.runs.get(ctx, req.RunID)
|
|
369
|
+
if errors.Is(err, errRunNotFound) {
|
|
370
|
+
// A retained newer attempt can outlive an old attempt row. Resolve the
|
|
371
|
+
// stable logical ID and acknowledge the old attempt as a stale
|
|
372
|
+
// duplicate; it must never be validated or signaled into the new run.
|
|
373
|
+
logicalID := run.ID(identity.LogicalRunID(req.RunID))
|
|
374
|
+
if latest, lookupErr := e.runs.findByLogicalID(ctx, logicalID); lookupErr == nil && latest.ID != req.RunID {
|
|
375
|
+
slog.Info("report duplicate ack", "ticket", latest.Ticket.Key,
|
|
376
|
+
"runID", string(req.RunID), "logicalRunID", string(logicalID),
|
|
377
|
+
"node", req.Node, "reportID", req.ReportID, "state", string(latest.State))
|
|
378
|
+
return run.ReportAck{Accepted: true, Duplicate: true}, nil
|
|
379
|
+
}
|
|
380
|
+
}
|
|
361
381
|
if err != nil {
|
|
362
382
|
return run.ReportAck{}, fmt.Errorf("resolve run %s: %w", req.RunID, err)
|
|
363
383
|
}
|
|
@@ -71,7 +71,7 @@ func TestMailboxDescriptionAndLaunchPromptAreTaskSystemNeutral(t *testing.T) {
|
|
|
71
71
|
if err != nil {
|
|
72
72
|
t.Fatal(err)
|
|
73
73
|
}
|
|
74
|
-
want := "Task system: " + taskSystem + "\nUse the " + taskSystem + " tools to read the parent ticket PAY-101.\n\nYour mailbox is PAY-234. Read its description and comments for node instructions and feedback.\n\
|
|
74
|
+
want := "Task system: " + taskSystem + "\nUse the " + taskSystem + " tools to read the parent ticket PAY-101.\n\nYour mailbox is PAY-234. Read its description and comments for node instructions and feedback.\n\nReturn the complete report directly. Relay-flow will show a native TUI approval dialog after the report is valid. Do not use OpenCode's Question tool for relay-flow approval."
|
|
75
75
|
if prompt != want {
|
|
76
76
|
t.Fatalf("RenderPrompt(%s) = %q, want %q", taskSystem, prompt, want)
|
|
77
77
|
}
|
|
@@ -82,6 +82,7 @@ type fakeTaskSystem struct {
|
|
|
82
82
|
completeFail int // next N CompleteMailbox calls return transient error
|
|
83
83
|
completeConflict bool // CompleteMailbox returns retry.ConflictError
|
|
84
84
|
completeSlow time.Duration // CompleteMailbox sleeps (a running activity)
|
|
85
|
+
startConflict bool // start ApplyTaskConfig returns a status conflict
|
|
85
86
|
failComments bool // Comment returns transient error
|
|
86
87
|
|
|
87
88
|
// Recovery fixtures.
|
|
@@ -165,6 +166,15 @@ func (s *fakeTaskSystem) EnsureMailboxes(_ context.Context, parent task.TicketRe
|
|
|
165
166
|
|
|
166
167
|
func (s *fakeTaskSystem) ApplyTaskConfig(_ context.Context, target task.Target, cfg config.RawValues) error {
|
|
167
168
|
key := target.Parent.Key
|
|
169
|
+
if target.Mailbox == nil {
|
|
170
|
+
s.mu.Lock()
|
|
171
|
+
conflict := s.startConflict
|
|
172
|
+
s.mu.Unlock()
|
|
173
|
+
if conflict {
|
|
174
|
+
s.log.add("applyTaskConfigConflict:" + key)
|
|
175
|
+
return retry.ConflictError(errStartConflict)
|
|
176
|
+
}
|
|
177
|
+
}
|
|
168
178
|
if target.Mailbox != nil {
|
|
169
179
|
key = target.Mailbox.Key
|
|
170
180
|
s.mu.Lock()
|
|
@@ -227,6 +237,16 @@ func (s *fakeTaskSystem) Comment(_ context.Context, target task.Target, body, ma
|
|
|
227
237
|
return nil
|
|
228
238
|
}
|
|
229
239
|
|
|
240
|
+
func (s *fakeTaskSystem) PrepareRestart(_ context.Context, parent task.TicketRef, mbs []task.Mailbox) error {
|
|
241
|
+
s.mu.Lock()
|
|
242
|
+
for _, mb := range mbs {
|
|
243
|
+
s.mailboxStatus[mb.Key] = "To Do"
|
|
244
|
+
}
|
|
245
|
+
s.mu.Unlock()
|
|
246
|
+
s.log.add("prepareRestart:" + parent.Key)
|
|
247
|
+
return nil
|
|
248
|
+
}
|
|
249
|
+
|
|
230
250
|
func (s *fakeTaskSystem) ResetForRecovery(_ context.Context, parent task.TicketRef, mbs []task.Mailbox, _ config.RawValues) error {
|
|
231
251
|
s.mu.Lock()
|
|
232
252
|
for _, mb := range mbs {
|
|
@@ -287,6 +307,12 @@ func (s *fakeTaskSystem) setMailboxStatus(key, status string) {
|
|
|
287
307
|
s.mu.Unlock()
|
|
288
308
|
}
|
|
289
309
|
|
|
310
|
+
func (s *fakeTaskSystem) setStartConflict(value bool) {
|
|
311
|
+
s.mu.Lock()
|
|
312
|
+
s.startConflict = value
|
|
313
|
+
s.mu.Unlock()
|
|
314
|
+
}
|
|
315
|
+
|
|
290
316
|
// seedMailbox pre-populates an existing mailbox (with labels) for recovery
|
|
291
317
|
// and reuse tests.
|
|
292
318
|
func (s *fakeTaskSystem) seedMailbox(parentKey string, mb task.Mailbox, labels []string) {
|
|
@@ -345,11 +371,12 @@ func (f *fakeRunner) ValidateRepo(context.Context, string, string) error { retur
|
|
|
345
371
|
func (f *fakeRunner) EnsureEnvironment(_ context.Context, spec runner.RunSpec) (runner.Environment, error) {
|
|
346
372
|
f.mu.Lock()
|
|
347
373
|
defer f.mu.Unlock()
|
|
348
|
-
|
|
374
|
+
key := spec.TicketKey
|
|
375
|
+
if e, ok := f.envs[key]; ok {
|
|
349
376
|
return e, nil
|
|
350
377
|
}
|
|
351
|
-
e := runner.Environment{ID: "env-" +
|
|
352
|
-
f.envs[
|
|
378
|
+
e := runner.Environment{ID: "env-" + spec.TicketKey, Path: spec.RepoPath}
|
|
379
|
+
f.envs[key] = e
|
|
353
380
|
f.log.add("ensureEnvironment:" + string(spec.RunID))
|
|
354
381
|
return e, nil
|
|
355
382
|
}
|
|
@@ -418,7 +445,7 @@ func (f *fakeRunner) CloseTerminal(_ context.Context, t runner.Terminal) error {
|
|
|
418
445
|
func (f *fakeRunner) CloseTerminals(_ context.Context, spec runner.RunSpec) error {
|
|
419
446
|
f.mu.Lock()
|
|
420
447
|
defer f.mu.Unlock()
|
|
421
|
-
prefix := "env-" +
|
|
448
|
+
prefix := "env-" + spec.TicketKey + "/"
|
|
422
449
|
for k, ft := range f.terminals {
|
|
423
450
|
if len(k) >= len(prefix) && k[:len(prefix)] == prefix {
|
|
424
451
|
ft.live = false
|
|
@@ -433,13 +460,13 @@ func (f *fakeRunner) CloseTerminals(_ context.Context, spec runner.RunSpec) erro
|
|
|
433
460
|
func (f *fakeRunner) CleanupRun(_ context.Context, spec runner.RunSpec) error {
|
|
434
461
|
f.mu.Lock()
|
|
435
462
|
defer f.mu.Unlock()
|
|
436
|
-
prefix := "env-" +
|
|
463
|
+
prefix := "env-" + spec.TicketKey + "/"
|
|
437
464
|
for k := range f.terminals {
|
|
438
465
|
if len(k) >= len(prefix) && k[:len(prefix)] == prefix {
|
|
439
466
|
delete(f.terminals, k)
|
|
440
467
|
}
|
|
441
468
|
}
|
|
442
|
-
delete(f.envs,
|
|
469
|
+
delete(f.envs, spec.TicketKey)
|
|
443
470
|
f.cleaned = append(f.cleaned, string(spec.RunID))
|
|
444
471
|
f.log.add("cleanupRun:" + string(spec.RunID))
|
|
445
472
|
return nil
|
|
@@ -553,3 +580,4 @@ type conflictError struct{ msg string }
|
|
|
553
580
|
func (e *conflictError) Error() string { return e.msg }
|
|
554
581
|
|
|
555
582
|
var errConflict = &conflictError{msg: "human moved mailbox"}
|
|
583
|
+
var errStartConflict = &conflictError{msg: "human moved ticket status"}
|
|
@@ -57,10 +57,18 @@ func jitter(ctx goworkflow.Context) (float64, error) {
|
|
|
57
57
|
// deterministic. On cancellation it runs cancellation cleanup on a
|
|
58
58
|
// disconnected context.
|
|
59
59
|
func (a *Activities) TicketWorkflow(ctx goworkflow.Context, start run.Start) error {
|
|
60
|
+
if start.LogicalID == "" {
|
|
61
|
+
start.LogicalID = run.ID(identity.LogicalRunID(start.ID))
|
|
62
|
+
}
|
|
63
|
+
if start.AttemptID == 0 {
|
|
64
|
+
start.AttemptID = 1
|
|
65
|
+
}
|
|
60
66
|
err := a.runGraph(ctx, start)
|
|
61
67
|
if err != nil && ctx.Err() != nil {
|
|
62
68
|
work := run.Work{
|
|
63
69
|
RunID: start.ID,
|
|
70
|
+
LogicalID: start.LogicalID,
|
|
71
|
+
AttemptID: start.AttemptID,
|
|
64
72
|
Repo: start.Repo,
|
|
65
73
|
Workflow: start.Workflow.Name,
|
|
66
74
|
Parent: start.Ticket,
|
|
@@ -76,6 +84,8 @@ func (a *Activities) runGraph(ctx goworkflow.Context, start run.Start) error {
|
|
|
76
84
|
wf := start.Workflow // value snapshot
|
|
77
85
|
work := run.Work{
|
|
78
86
|
RunID: start.ID,
|
|
87
|
+
LogicalID: start.LogicalID,
|
|
88
|
+
AttemptID: start.AttemptID,
|
|
79
89
|
Repo: start.Repo,
|
|
80
90
|
Workflow: wf.Name,
|
|
81
91
|
Parent: start.Ticket,
|
|
@@ -93,6 +103,26 @@ func (a *Activities) runGraph(ctx goworkflow.Context, start run.Start) error {
|
|
|
93
103
|
return err
|
|
94
104
|
}
|
|
95
105
|
|
|
106
|
+
// An explicit restart reuses the task-system mailboxes and ticket
|
|
107
|
+
// worktree, but resets relay-owned mailbox state and closes stale node
|
|
108
|
+
// terminals before the fresh start edge is processed. Human-owned
|
|
109
|
+
// incompatible states are returned as conflicts and keep this attempt
|
|
110
|
+
// blocked until the human restores a compatible state.
|
|
111
|
+
if start.AttemptID > 1 {
|
|
112
|
+
mailboxList := make([]task.Mailbox, 0, len(mailboxes))
|
|
113
|
+
for _, mailbox := range mailboxes {
|
|
114
|
+
mailboxList = append(mailboxList, mailbox)
|
|
115
|
+
}
|
|
116
|
+
sort.Slice(mailboxList, func(i, j int) bool { return mailboxList[i].Node < mailboxList[j].Node })
|
|
117
|
+
if _, err := retryLoop(ctx, start.ID, a, work, "start",
|
|
118
|
+
func(ctx2 goworkflow.Context) goworkflow.Future[struct{}] {
|
|
119
|
+
return goworkflow.ExecuteActivity[struct{}](ctx2, noNativeRetries,
|
|
120
|
+
a.PrepareRestart, work, start.RepoPath, mailboxList)
|
|
121
|
+
}); err != nil {
|
|
122
|
+
return err
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
96
126
|
// Validate every referenced agent before the start edge.
|
|
97
127
|
agentSet := map[string]bool{}
|
|
98
128
|
for _, n := range wf.Nodes {
|
|
@@ -443,6 +473,7 @@ func retryLoop[T any](ctx goworkflow.Context, id run.ID, a *Activities, work run
|
|
|
443
473
|
}
|
|
444
474
|
f := classifyActivityError(err)
|
|
445
475
|
if f.Kind == retry.Conflict {
|
|
476
|
+
f.Message = blockedMessage(work, node, f.Message)
|
|
446
477
|
// Mark blocked, keep retrying on the capped schedule.
|
|
447
478
|
_, _ = scheduleState(ctx, a, id, run.StateBlocked, f.Message)
|
|
448
479
|
blocked = true
|
|
@@ -463,6 +494,18 @@ func retryLoop[T any](ctx goworkflow.Context, id run.ID, a *Activities, work run
|
|
|
463
494
|
}
|
|
464
495
|
}
|
|
465
496
|
|
|
497
|
+
func blockedMessage(work run.Work, node, message string) string {
|
|
498
|
+
message = strings.TrimRight(message, ". ")
|
|
499
|
+
lower := strings.ToLower(message)
|
|
500
|
+
if node == "start" && !strings.Contains(lower, "mailbox") {
|
|
501
|
+
return fmt.Sprintf("%s. Move ticket %s to an allowed active start status; relay-flow will retry automatically", message, work.Parent.Key)
|
|
502
|
+
}
|
|
503
|
+
if node != "" {
|
|
504
|
+
return fmt.Sprintf("%s. Restore the task-system state required for node %s; relay-flow will retry automatically", message, node)
|
|
505
|
+
}
|
|
506
|
+
return fmt.Sprintf("%s. Restore the task-system state required by this operation; relay-flow will retry automatically", message)
|
|
507
|
+
}
|
|
508
|
+
|
|
466
509
|
// logRetry emits the 9.6 retry-classification info line, replay-safe.
|
|
467
510
|
// Attrs come from the always-known run.Work value carried by the caller,
|
|
468
511
|
// so ticket/repo/workflow are present even if the projection is briefly
|
|
@@ -565,6 +608,10 @@ func mustJitter(ctx goworkflow.Context) float64 {
|
|
|
565
608
|
// canceled. No rollback/compensation ever runs.
|
|
566
609
|
func (a *Activities) cancelCleanup(ctx goworkflow.Context, work run.Work, repoPath, reason string) error {
|
|
567
610
|
dctx := goworkflow.NewDisconnectedContext(ctx)
|
|
611
|
+
markerID := work.LogicalID
|
|
612
|
+
if markerID == "" {
|
|
613
|
+
markerID = work.RunID
|
|
614
|
+
}
|
|
568
615
|
if _, err := retryLoop(dctx, work.RunID, a, work, "",
|
|
569
616
|
func(ctx2 goworkflow.Context) goworkflow.Future[struct{}] {
|
|
570
617
|
return goworkflow.ExecuteActivity[struct{}](ctx2, noNativeRetries,
|
|
@@ -578,7 +625,7 @@ func (a *Activities) cancelCleanup(ctx goworkflow.Context, work run.Work, repoPa
|
|
|
578
625
|
RunID: work.RunID,
|
|
579
626
|
Item: task.Target{Parent: work.Parent},
|
|
580
627
|
Body: "Run canceled: " + reason,
|
|
581
|
-
Marker: run.CancellationMarker(
|
|
628
|
+
Marker: run.CancellationMarker(markerID),
|
|
582
629
|
})
|
|
583
630
|
}); err != nil {
|
|
584
631
|
return err
|
|
@@ -34,6 +34,8 @@ type NodeRuntime struct {
|
|
|
34
34
|
const relayRunsSchema = `
|
|
35
35
|
CREATE TABLE IF NOT EXISTS relay_runs (
|
|
36
36
|
id TEXT PRIMARY KEY,
|
|
37
|
+
logical_run_id TEXT,
|
|
38
|
+
attempt_id INTEGER,
|
|
37
39
|
repo TEXT NOT NULL,
|
|
38
40
|
workflow TEXT NOT NULL,
|
|
39
41
|
ticket_id TEXT NOT NULL,
|
|
@@ -86,9 +88,11 @@ func (p *RunProjection) migrate() error {
|
|
|
86
88
|
return err
|
|
87
89
|
}
|
|
88
90
|
for name, definition := range map[string]string{
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
91
|
+
"logical_run_id": "TEXT",
|
|
92
|
+
"attempt_id": "INTEGER",
|
|
93
|
+
"retry_error": "TEXT",
|
|
94
|
+
"retry_attempt": "INTEGER",
|
|
95
|
+
"next_retry_at": "DATETIME",
|
|
92
96
|
} {
|
|
93
97
|
var count int
|
|
94
98
|
if err := p.DB.QueryRow(`SELECT COUNT(1) FROM pragma_table_info('relay_runs') WHERE name = ?`, name).Scan(&count); err != nil {
|
|
@@ -100,6 +104,15 @@ func (p *RunProjection) migrate() error {
|
|
|
100
104
|
}
|
|
101
105
|
}
|
|
102
106
|
}
|
|
107
|
+
// Rows created before attempt identities were introduced represent the
|
|
108
|
+
// original attempt. Backfill the stable logical ID and attempt number so
|
|
109
|
+
// restart allocation remains numeric and never reuses attempt 1.
|
|
110
|
+
if _, err := p.DB.Exec(`UPDATE relay_runs SET logical_run_id = id WHERE COALESCE(logical_run_id, '') = ''`); err != nil {
|
|
111
|
+
return err
|
|
112
|
+
}
|
|
113
|
+
if _, err := p.DB.Exec(`UPDATE relay_runs SET attempt_id = 1 WHERE attempt_id IS NULL OR attempt_id = 0`); err != nil {
|
|
114
|
+
return err
|
|
115
|
+
}
|
|
103
116
|
return nil
|
|
104
117
|
}
|
|
105
118
|
|
|
@@ -110,11 +123,19 @@ var errNodeRuntimeNotFound = errors.New("node runtime not found")
|
|
|
110
123
|
func IsNotFound(err error) bool { return errors.Is(err, errRunNotFound) }
|
|
111
124
|
|
|
112
125
|
func (p *RunProjection) insertStart(ctx context.Context, s run.Start, now time.Time) error {
|
|
126
|
+
logicalID := s.LogicalID
|
|
127
|
+
if logicalID == "" {
|
|
128
|
+
logicalID = s.ID
|
|
129
|
+
}
|
|
130
|
+
attemptID := s.AttemptID
|
|
131
|
+
if attemptID == 0 {
|
|
132
|
+
attemptID = 1
|
|
133
|
+
}
|
|
113
134
|
_, err := p.DB.ExecContext(ctx, `
|
|
114
|
-
INSERT INTO relay_runs (id, repo, workflow, ticket_id, ticket_key, state, started_at, updated_at)
|
|
115
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
135
|
+
INSERT INTO relay_runs (id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, started_at, updated_at)
|
|
136
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
116
137
|
ON CONFLICT(id) DO NOTHING`,
|
|
117
|
-
string(s.ID), s.Repo, s.Workflow.Name, s.Ticket.ID, s.Ticket.Key,
|
|
138
|
+
string(s.ID), string(logicalID), int64(attemptID), s.Repo, s.Workflow.Name, s.Ticket.ID, s.Ticket.Key,
|
|
118
139
|
string(run.StateStarting), now, now)
|
|
119
140
|
return err
|
|
120
141
|
}
|
|
@@ -360,7 +381,7 @@ func nullableString(value string) any {
|
|
|
360
381
|
|
|
361
382
|
func (p *RunProjection) get(ctx context.Context, id run.ID) (run.Run, error) {
|
|
362
383
|
row := p.DB.QueryRowContext(ctx, `
|
|
363
|
-
SELECT id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
384
|
+
SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
364
385
|
retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at
|
|
365
386
|
FROM relay_runs WHERE id = ?`, string(id))
|
|
366
387
|
return scanRun(row)
|
|
@@ -368,9 +389,17 @@ func (p *RunProjection) get(ctx context.Context, id run.ID) (run.Run, error) {
|
|
|
368
389
|
|
|
369
390
|
func (p *RunProjection) findByTicket(ctx context.Context, ticket string) (run.Run, error) {
|
|
370
391
|
row := p.DB.QueryRowContext(ctx, `
|
|
371
|
-
SELECT id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
392
|
+
SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
372
393
|
retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at
|
|
373
|
-
FROM relay_runs WHERE ticket_key = ? ORDER BY started_at DESC LIMIT 1`, ticket)
|
|
394
|
+
FROM relay_runs WHERE ticket_key = ? ORDER BY started_at DESC, attempt_id DESC LIMIT 1`, ticket)
|
|
395
|
+
return scanRun(row)
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
func (p *RunProjection) findByLogicalID(ctx context.Context, logicalID run.ID) (run.Run, error) {
|
|
399
|
+
row := p.DB.QueryRowContext(ctx, `
|
|
400
|
+
SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error,
|
|
401
|
+
retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at
|
|
402
|
+
FROM relay_runs WHERE logical_run_id = ? ORDER BY started_at DESC, attempt_id DESC LIMIT 1`, string(logicalID))
|
|
374
403
|
return scanRun(row)
|
|
375
404
|
}
|
|
376
405
|
|
|
@@ -380,11 +409,13 @@ type rowScanner interface {
|
|
|
380
409
|
|
|
381
410
|
func scanRun(row rowScanner) (run.Run, error) {
|
|
382
411
|
var r run.Run
|
|
412
|
+
var logicalID sql.NullString
|
|
413
|
+
var attemptNumber sql.NullInt64
|
|
383
414
|
var node, visit, lastErr, retryErr sql.NullString
|
|
384
415
|
var retryAttempt sql.NullInt64
|
|
385
416
|
var nextRetry, finished sql.NullTime
|
|
386
417
|
var started, updated time.Time
|
|
387
|
-
err := row.Scan(&r.ID, &r.Repo, &r.Workflow, &r.Ticket.ID, &r.Ticket.Key, &r.State,
|
|
418
|
+
err := row.Scan(&r.ID, &logicalID, &attemptNumber, &r.Repo, &r.Workflow, &r.Ticket.ID, &r.Ticket.Key, &r.State,
|
|
388
419
|
&node, &visit, &lastErr, &retryErr, &retryAttempt, &nextRetry, &started, &updated, &finished)
|
|
389
420
|
if errors.Is(err, sql.ErrNoRows) {
|
|
390
421
|
return run.Run{}, errRunNotFound
|
|
@@ -392,6 +423,16 @@ func scanRun(row rowScanner) (run.Run, error) {
|
|
|
392
423
|
if err != nil {
|
|
393
424
|
return run.Run{}, err
|
|
394
425
|
}
|
|
426
|
+
if logicalID.Valid && logicalID.String != "" {
|
|
427
|
+
r.LogicalID = run.ID(logicalID.String)
|
|
428
|
+
} else {
|
|
429
|
+
r.LogicalID = r.ID
|
|
430
|
+
}
|
|
431
|
+
if attemptNumber.Valid && attemptNumber.Int64 > 0 {
|
|
432
|
+
r.AttemptID = run.AttemptID(attemptNumber.Int64)
|
|
433
|
+
} else {
|
|
434
|
+
r.AttemptID = 1
|
|
435
|
+
}
|
|
395
436
|
r.CurrentNode = node.String
|
|
396
437
|
r.CurrentNodeVisitID = run.NodeVisitID(visit.String)
|
|
397
438
|
r.LastError = lastErr.String
|
|
@@ -410,7 +451,7 @@ func scanRun(row rowScanner) (run.Run, error) {
|
|
|
410
451
|
}
|
|
411
452
|
|
|
412
453
|
func (p *RunProjection) list(ctx context.Context, f run.Filter) ([]run.Run, error) {
|
|
413
|
-
q := `SELECT id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error, retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at FROM relay_runs WHERE 1=1`
|
|
454
|
+
q := `SELECT id, logical_run_id, attempt_id, repo, workflow, ticket_id, ticket_key, state, current_node, current_node_visit_id, last_error, retry_error, retry_attempt, next_retry_at, started_at, updated_at, finished_at FROM relay_runs WHERE 1=1`
|
|
414
455
|
var args []any
|
|
415
456
|
if f.Repo != "" {
|
|
416
457
|
q += ` AND repo = ?`
|
|
@@ -5,6 +5,7 @@ import (
|
|
|
5
5
|
"database/sql"
|
|
6
6
|
"os"
|
|
7
7
|
"path/filepath"
|
|
8
|
+
"strings"
|
|
8
9
|
"testing"
|
|
9
10
|
"time"
|
|
10
11
|
|
|
@@ -219,6 +220,131 @@ func TestCancelRun(t *testing.T) {
|
|
|
219
220
|
}
|
|
220
221
|
}
|
|
221
222
|
|
|
223
|
+
func TestExplicitRestartCreatesFreshAttemptFromStart(t *testing.T) {
|
|
224
|
+
log := newEventLog()
|
|
225
|
+
sys := newFakeTaskSystem(log)
|
|
226
|
+
fr := newFakeRunner(log)
|
|
227
|
+
fh := newFakeHarness(log)
|
|
228
|
+
repos := repoRegistryWith("payments", sys)
|
|
229
|
+
wf := linearWorkflow(false)
|
|
230
|
+
workflows := &workflow.Registry{}
|
|
231
|
+
workflows.Replace(&wf)
|
|
232
|
+
engine := newEngine(t, goworkflows.Dependencies{
|
|
233
|
+
Repos: repos, Runner: fr, Harness: fh,
|
|
234
|
+
})
|
|
235
|
+
oldID, err := startRun(engine, wf)
|
|
236
|
+
if err != nil {
|
|
237
|
+
t.Fatal(err)
|
|
238
|
+
}
|
|
239
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
240
|
+
r, _ := engine.GetRun(context.Background(), oldID)
|
|
241
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
242
|
+
})
|
|
243
|
+
if err := engine.CancelRun(context.Background(), oldID, "operator requested restart"); err != nil {
|
|
244
|
+
t.Fatal(err)
|
|
245
|
+
}
|
|
246
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
247
|
+
r, _ := engine.GetRun(context.Background(), oldID)
|
|
248
|
+
return r.State == run.StateCanceled
|
|
249
|
+
})
|
|
250
|
+
|
|
251
|
+
manager := &run.RunManager{
|
|
252
|
+
Executor: engine, Runs: engine, Repos: repos, Workflows: workflows,
|
|
253
|
+
}
|
|
254
|
+
fresh, err := manager.RestartByTicket(context.Background(), "PAY-101")
|
|
255
|
+
if err != nil {
|
|
256
|
+
t.Fatalf("RestartByTicket failed: %v", err)
|
|
257
|
+
}
|
|
258
|
+
if fresh.ID == oldID || fresh.LogicalID != oldID || fresh.AttemptID != 2 {
|
|
259
|
+
t.Fatalf("fresh attempt = %+v, want logical=%q attempt=2 and a new ID", fresh, oldID)
|
|
260
|
+
}
|
|
261
|
+
oldAck, err := engine.SubmitReport(context.Background(), reportRequest(oldID, "coding", successReport("end")))
|
|
262
|
+
if err != nil || !oldAck.Accepted || !oldAck.Duplicate {
|
|
263
|
+
t.Fatalf("stale old-attempt report ack=%+v err=%v, want accepted duplicate", oldAck, err)
|
|
264
|
+
}
|
|
265
|
+
if got := string(fresh.ID); got != string(oldID)+"~attempt~2" {
|
|
266
|
+
t.Fatalf("fresh execution ID = %q, want numeric attempt suffix", got)
|
|
267
|
+
}
|
|
268
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
269
|
+
r, _ := engine.GetRun(context.Background(), fresh.ID)
|
|
270
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
271
|
+
})
|
|
272
|
+
|
|
273
|
+
current, err := engine.GetRun(context.Background(), fresh.ID)
|
|
274
|
+
if err != nil {
|
|
275
|
+
t.Fatal(err)
|
|
276
|
+
}
|
|
277
|
+
latest, err := engine.FindRunByTicket(context.Background(), "PAY-101")
|
|
278
|
+
if err != nil {
|
|
279
|
+
t.Fatal(err)
|
|
280
|
+
}
|
|
281
|
+
if latest.ID != fresh.ID || latest.AttemptID != 2 || latest.LogicalID != oldID {
|
|
282
|
+
t.Fatalf("ticket lookup = %+v, want latest fresh attempt %q", latest, fresh.ID)
|
|
283
|
+
}
|
|
284
|
+
if current.State != run.StateWaiting && current.State != run.StateRunning {
|
|
285
|
+
t.Fatalf("fresh attempt state = %q, want active node state", current.State)
|
|
286
|
+
}
|
|
287
|
+
if got := log.count("prepareRestart:PAY-101"); got != 1 {
|
|
288
|
+
t.Fatalf("restart preparation calls = %d, want 1", got)
|
|
289
|
+
}
|
|
290
|
+
if len(fr.envs) != 1 {
|
|
291
|
+
t.Fatalf("restart created a second ticket environment: %d", len(fr.envs))
|
|
292
|
+
}
|
|
293
|
+
if fr.liveTerminals() != 1 {
|
|
294
|
+
t.Fatalf("restart left %d live terminals, want one fresh node terminal", fr.liveTerminals())
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
func TestRestartStatusConflictIsVisibleAndRecovers(t *testing.T) {
|
|
299
|
+
log := newEventLog()
|
|
300
|
+
sys := newFakeTaskSystem(log)
|
|
301
|
+
fr := newFakeRunner(log)
|
|
302
|
+
repos := repoRegistryWith("payments", sys)
|
|
303
|
+
wf := linearWorkflow(false)
|
|
304
|
+
workflows := &workflow.Registry{}
|
|
305
|
+
workflows.Replace(&wf)
|
|
306
|
+
engine := newEngine(t, goworkflows.Dependencies{Repos: repos, Runner: fr, Harness: newFakeHarness(log)})
|
|
307
|
+
oldID, err := startRun(engine, wf)
|
|
308
|
+
if err != nil {
|
|
309
|
+
t.Fatal(err)
|
|
310
|
+
}
|
|
311
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
312
|
+
r, _ := engine.GetRun(context.Background(), oldID)
|
|
313
|
+
return r.CurrentNode == "coding"
|
|
314
|
+
})
|
|
315
|
+
if err := engine.CancelRun(context.Background(), oldID, "operator requested restart"); err != nil {
|
|
316
|
+
t.Fatal(err)
|
|
317
|
+
}
|
|
318
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
319
|
+
r, _ := engine.GetRun(context.Background(), oldID)
|
|
320
|
+
return r.State == run.StateCanceled
|
|
321
|
+
})
|
|
322
|
+
|
|
323
|
+
sys.setStartConflict(true)
|
|
324
|
+
manager := &run.RunManager{Executor: engine, Runs: engine, Repos: repos, Workflows: workflows}
|
|
325
|
+
fresh, err := manager.RestartByTicket(context.Background(), "PAY-101")
|
|
326
|
+
if err != nil {
|
|
327
|
+
t.Fatal(err)
|
|
328
|
+
}
|
|
329
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
330
|
+
r, _ := engine.GetRun(context.Background(), fresh.ID)
|
|
331
|
+
return r.State == run.StateBlocked
|
|
332
|
+
})
|
|
333
|
+
blocked, err := engine.GetRun(context.Background(), fresh.ID)
|
|
334
|
+
if err != nil {
|
|
335
|
+
t.Fatal(err)
|
|
336
|
+
}
|
|
337
|
+
if !strings.Contains(blocked.LastError, "Move ticket PAY-101 to an allowed active start status") {
|
|
338
|
+
t.Fatalf("blocked LastError = %q, want actionable start-status guidance", blocked.LastError)
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
sys.setStartConflict(false)
|
|
342
|
+
waitFor(t, 60*time.Second, func() bool {
|
|
343
|
+
r, _ := engine.GetRun(context.Background(), fresh.ID)
|
|
344
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
345
|
+
})
|
|
346
|
+
}
|
|
347
|
+
|
|
222
348
|
func TestCancelDuringRunningActivity(t *testing.T) {
|
|
223
349
|
// Cancellation cannot interrupt an already-running activity; it waits
|
|
224
350
|
// for it to return, then runs cancellation cleanup.
|
|
@@ -444,6 +570,9 @@ func TestConflictMarksBlockedThenRecovers(t *testing.T) {
|
|
|
444
570
|
if r.LastError == "" {
|
|
445
571
|
t.Fatal("blocked run exposes no conflict error in LastError")
|
|
446
572
|
}
|
|
573
|
+
if !strings.Contains(r.LastError, "Restore the task-system state required for node coding") {
|
|
574
|
+
t.Fatalf("blocked LastError = %q, want actionable node-state guidance", r.LastError)
|
|
575
|
+
}
|
|
447
576
|
if sys.mailboxStatusOf("PAY-101-coding") == "Done" {
|
|
448
577
|
t.Fatal("mailbox completed while state was incompatible; no blind overwrite allowed")
|
|
449
578
|
}
|