relay-flow 0.2.2-alpha → 0.2.3-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -3
- package/cmd/relay-flow/commands_test.go +5 -1
- package/cmd/relay-flow/main.go +25 -1
- package/cmd/relay-flow/serve.go +12 -0
- package/internal/execution/goworkflows/activities.go +19 -0
- package/internal/execution/goworkflows/engine.go +20 -0
- package/internal/execution/goworkflows/engine_test.go +1 -1
- package/internal/execution/goworkflows/fakes_test.go +34 -6
- package/internal/execution/goworkflows/interpreter.go +48 -1
- package/internal/execution/goworkflows/projection.go +52 -11
- package/internal/execution/goworkflows/recovery_test.go +129 -0
- package/internal/harness/opencode/opencode.go +4 -4
- package/internal/harness/opencode/opencode_test.go +59 -4
- package/internal/harness/opencode/repo_setup.go +42 -2
- package/internal/identity/identity.go +28 -1
- package/internal/identity/identity_test.go +40 -0
- package/internal/run/manager.go +193 -25
- package/internal/run/run.go +24 -6
- package/internal/run/run_manager_test.go +100 -0
- package/internal/server/api_test.go +54 -0
- package/internal/server/client.go +11 -0
- package/internal/server/fixture_test.go +18 -0
- package/internal/server/server.go +16 -0
- package/internal/task/beads/beads.go +16 -0
- package/internal/task/beads/status_compatibility_test.go +43 -0
- package/internal/task/jira/helpers_test.go +3 -0
- package/internal/task/jira/jira.go +16 -0
- package/internal/task/jira/transition_defaults_test.go +43 -0
- package/internal/task/task.go +8 -0
- package/package.json +1 -1
package/internal/run/manager.go
CHANGED
|
@@ -5,6 +5,7 @@ import (
|
|
|
5
5
|
"fmt"
|
|
6
6
|
"log/slog"
|
|
7
7
|
"sync"
|
|
8
|
+
"time"
|
|
8
9
|
|
|
9
10
|
"github.com/rajpopat27/relay-flow/internal/identity"
|
|
10
11
|
"github.com/rajpopat27/relay-flow/internal/repo"
|
|
@@ -13,12 +14,15 @@ import (
|
|
|
13
14
|
)
|
|
14
15
|
|
|
15
16
|
// CancellationMarker is the stable parent comment marker recording that the
|
|
16
|
-
// run was canceled; a missing claimed run carrying it is never
|
|
17
|
+
// logical run was canceled; a missing claimed run carrying it is never
|
|
18
|
+
// recreated by normal polling.
|
|
17
19
|
func CancellationMarker(id ID) string {
|
|
18
20
|
return string(id) + ":cancellation"
|
|
19
21
|
}
|
|
20
22
|
|
|
21
|
-
// RunManager performs
|
|
23
|
+
// RunManager performs assignment and durable-run creation. Repos and
|
|
24
|
+
// Workflows are only needed by the explicit restart operation; normal poll
|
|
25
|
+
// handling continues to receive the already-resolved repo/workflow values.
|
|
22
26
|
type RunManager struct {
|
|
23
27
|
Executor Executor
|
|
24
28
|
Runs RunQueries
|
|
@@ -28,12 +32,37 @@ type RunManager struct {
|
|
|
28
32
|
// a run never starts against a workflow definition that is concurrently
|
|
29
33
|
// being replaced or removed. A plain *sync.Mutex — no lock service.
|
|
30
34
|
Gate *sync.Mutex
|
|
35
|
+
|
|
36
|
+
// Repos and Workflows resolve the current task-system repo and latest
|
|
37
|
+
// workflow snapshot for an explicit restart. They are concrete registries,
|
|
38
|
+
// not task/runner/harness-specific dependencies.
|
|
39
|
+
Repos *repo.Registry
|
|
40
|
+
Workflows *workflow.Registry
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
func terminalState(state State) bool {
|
|
44
|
+
return state == StateCompleted || state == StateCanceled
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
func activeState(state State) bool {
|
|
48
|
+
return !terminalState(state) && state != StateCanceling
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
func newerRun(candidate, current Run) bool {
|
|
52
|
+
if candidate.StartedAt.After(current.StartedAt) {
|
|
53
|
+
return true
|
|
54
|
+
}
|
|
55
|
+
if candidate.StartedAt.Equal(current.StartedAt) && candidate.UpdatedAt.After(current.UpdatedAt) {
|
|
56
|
+
return true
|
|
57
|
+
}
|
|
58
|
+
return false
|
|
31
59
|
}
|
|
32
60
|
|
|
33
61
|
// EnsureRun claims the ticket if unassigned, skips claiming when the ticket
|
|
34
|
-
// is already assigned to this workflow,
|
|
35
|
-
//
|
|
36
|
-
// run with a value snapshot of
|
|
62
|
+
// is already assigned to this workflow, reuses the newest active execution
|
|
63
|
+
// attempt, checks the stable logical cancellation marker before recreating a
|
|
64
|
+
// missing claimed run, then ensures the durable run with a value snapshot of
|
|
65
|
+
// the workflow.
|
|
37
66
|
func (m *RunManager) EnsureRun(ctx context.Context, rp *repo.Repo, wf *workflow.Workflow, ticket task.Ticket) error {
|
|
38
67
|
if m.Gate != nil {
|
|
39
68
|
m.Gate.Lock()
|
|
@@ -47,19 +76,51 @@ func (m *RunManager) EnsureRun(ctx context.Context, rp *repo.Repo, wf *workflow.
|
|
|
47
76
|
break
|
|
48
77
|
}
|
|
49
78
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
79
|
+
|
|
80
|
+
var candidates []Run
|
|
81
|
+
if m.Runs != nil {
|
|
82
|
+
var err error
|
|
83
|
+
candidates, err = m.Runs.ListRuns(ctx, Filter{Repo: rp.Name, Workflow: wf.Name, Ticket: ticket.Key})
|
|
53
84
|
if err != nil {
|
|
54
85
|
return fmt.Errorf("check existing run %s: %w", id, err)
|
|
55
86
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
87
|
+
}
|
|
88
|
+
var latest Run
|
|
89
|
+
latestSet := false
|
|
90
|
+
var active *Run
|
|
91
|
+
for _, candidate := range candidates {
|
|
92
|
+
if candidate.LogicalID == "" {
|
|
93
|
+
candidate.LogicalID = id
|
|
94
|
+
}
|
|
95
|
+
if candidate.AttemptID == 0 {
|
|
96
|
+
candidate.AttemptID = 1
|
|
97
|
+
}
|
|
98
|
+
if !latestSet || newerRun(candidate, latest) {
|
|
99
|
+
latest = candidate
|
|
100
|
+
latestSet = true
|
|
101
|
+
}
|
|
102
|
+
if activeState(candidate.State) && candidate.ID != "" && (active == nil || newerRun(candidate, *active)) {
|
|
103
|
+
copy := candidate
|
|
104
|
+
active = ©
|
|
61
105
|
}
|
|
62
106
|
}
|
|
107
|
+
|
|
108
|
+
// A restarted attempt (including a blocked one) is the current execution
|
|
109
|
+
// and must be ensured by its fenced ID. Do not inspect the cancellation
|
|
110
|
+
// marker for an active attempt.
|
|
111
|
+
if active != nil {
|
|
112
|
+
return m.ensure(ctx, Start{
|
|
113
|
+
ID: active.ID, LogicalID: active.LogicalID, AttemptID: active.AttemptID,
|
|
114
|
+
Repo: rp.Name, RepoPath: rp.Path, Workflow: *wf, Ticket: ticket.Ref(),
|
|
115
|
+
})
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Canceling/canceled/completed executions are terminal for normal polling.
|
|
119
|
+
// Only the explicit restart operation may create a new attempt.
|
|
120
|
+
if latestSet && (latest.State == StateCanceling || terminalState(latest.State)) {
|
|
121
|
+
return nil
|
|
122
|
+
}
|
|
123
|
+
|
|
63
124
|
if !claimed {
|
|
64
125
|
if err := rp.TaskSystem.Claim(ctx, ticket.Ref(), wf.Name); err != nil {
|
|
65
126
|
slog.Info("ensure-run outcome",
|
|
@@ -67,9 +128,9 @@ func (m *RunManager) EnsureRun(ctx context.Context, rp *repo.Repo, wf *workflow.
|
|
|
67
128
|
"outcome", "error", "stage", "claim", "error", err)
|
|
68
129
|
return fmt.Errorf("claim %s for workflow %s: %w", ticket.Key, wf.Name, err)
|
|
69
130
|
}
|
|
70
|
-
} else
|
|
71
|
-
// Claimed but
|
|
72
|
-
//
|
|
131
|
+
} else {
|
|
132
|
+
// Claimed but missing its run (claim-before-run crash gap or retention
|
|
133
|
+
// cleanup): never recreate a canceled logical run.
|
|
73
134
|
marked, err := rp.TaskSystem.HasComment(ctx, task.Target{Parent: ticket.Ref()}, CancellationMarker(id))
|
|
74
135
|
if err != nil {
|
|
75
136
|
slog.Info("ensure-run outcome",
|
|
@@ -84,29 +145,136 @@ func (m *RunManager) EnsureRun(ctx context.Context, rp *repo.Repo, wf *workflow.
|
|
|
84
145
|
return nil
|
|
85
146
|
}
|
|
86
147
|
}
|
|
87
|
-
|
|
88
|
-
ID:
|
|
89
|
-
Repo:
|
|
90
|
-
RepoPath: rp.Path,
|
|
91
|
-
Workflow: *wf,
|
|
92
|
-
Ticket: ticket.Ref(),
|
|
148
|
+
return m.ensure(ctx, Start{
|
|
149
|
+
ID: id, LogicalID: id, AttemptID: 1,
|
|
150
|
+
Repo: rp.Name, RepoPath: rp.Path, Workflow: *wf, Ticket: ticket.Ref(),
|
|
93
151
|
})
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
func (m *RunManager) ensure(ctx context.Context, start Start) error {
|
|
155
|
+
if m.Executor == nil {
|
|
156
|
+
return fmt.Errorf("ensure run %s: executor is not configured", start.ID)
|
|
157
|
+
}
|
|
158
|
+
created, err := m.Executor.EnsureRun(ctx, start)
|
|
94
159
|
if err != nil {
|
|
95
160
|
slog.Info("ensure-run outcome",
|
|
96
|
-
"ticket",
|
|
161
|
+
"ticket", start.Ticket.Key, "repo", start.Repo, "workflow", start.Workflow.Name, "runID", string(start.ID),
|
|
97
162
|
"outcome", "error", "stage", "executor", "error", err)
|
|
98
|
-
return fmt.Errorf("ensure run %s: %w",
|
|
163
|
+
return fmt.Errorf("ensure run %s: %w", start.ID, err)
|
|
99
164
|
}
|
|
100
165
|
outcome := "exists"
|
|
101
166
|
if created {
|
|
102
167
|
outcome = "created"
|
|
103
168
|
}
|
|
104
169
|
slog.Info("ensure-run outcome",
|
|
105
|
-
"ticket",
|
|
170
|
+
"ticket", start.Ticket.Key, "repo", start.Repo, "workflow", start.Workflow.Name, "runID", string(start.ID),
|
|
106
171
|
"outcome", outcome)
|
|
107
172
|
return nil
|
|
108
173
|
}
|
|
109
174
|
|
|
175
|
+
// RestartByTicket creates or returns the one active fresh attempt for a
|
|
176
|
+
// canceled ticket. It resolves the repo and latest workflow from the current
|
|
177
|
+
// registries, so the new durable snapshot is never copied from the canceled
|
|
178
|
+
// attempt. External task/runner/harness behavior remains behind their normal
|
|
179
|
+
// boundaries and is performed by the durable workflow after creation.
|
|
180
|
+
func (m *RunManager) RestartByTicket(ctx context.Context, ticket string) (Run, error) {
|
|
181
|
+
if m.Gate != nil {
|
|
182
|
+
m.Gate.Lock()
|
|
183
|
+
defer m.Gate.Unlock()
|
|
184
|
+
}
|
|
185
|
+
if m.Runs == nil {
|
|
186
|
+
return Run{}, fmt.Errorf("restart %s: run queries are not configured", ticket)
|
|
187
|
+
}
|
|
188
|
+
previous, err := m.Runs.FindRunByTicket(ctx, ticket)
|
|
189
|
+
if err != nil {
|
|
190
|
+
return Run{}, fmt.Errorf("find run for ticket %s: %w", ticket, err)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Repeating the command while the fresh attempt is active is idempotent.
|
|
194
|
+
// A canceling attempt is deliberately not treated as active: callers must
|
|
195
|
+
// wait for cancellation cleanup to finish before starting another attempt.
|
|
196
|
+
if activeState(previous.State) && previous.AttemptID != 0 {
|
|
197
|
+
return previous, nil
|
|
198
|
+
}
|
|
199
|
+
if previous.State == StateCanceling {
|
|
200
|
+
return Run{}, fmt.Errorf("%w: run %s is still canceling; wait for cancellation to finish", ErrRestartConflict, previous.ID)
|
|
201
|
+
}
|
|
202
|
+
if previous.State != StateCanceled {
|
|
203
|
+
return Run{}, fmt.Errorf("%w: run %s is %s; only canceled runs can be restarted", ErrRestartConflict, previous.ID, previous.State)
|
|
204
|
+
}
|
|
205
|
+
if m.Executor == nil || m.Repos == nil || m.Workflows == nil {
|
|
206
|
+
return Run{}, fmt.Errorf("restart %s: restart dependencies are not configured", ticket)
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
rp, ok := m.Repos.Get(previous.Repo)
|
|
210
|
+
if !ok {
|
|
211
|
+
return Run{}, fmt.Errorf("%w: repo %q for canceled run %s is no longer registered", ErrRestartConflict, previous.Repo, previous.ID)
|
|
212
|
+
}
|
|
213
|
+
wf, ok := m.Workflows.Get(previous.Workflow)
|
|
214
|
+
if !ok {
|
|
215
|
+
return Run{}, fmt.Errorf("%w: workflow %q for canceled run %s is no longer stored", ErrRestartConflict, previous.Workflow, previous.ID)
|
|
216
|
+
}
|
|
217
|
+
bound := false
|
|
218
|
+
for _, name := range wf.Repos {
|
|
219
|
+
if name == previous.Repo {
|
|
220
|
+
bound = true
|
|
221
|
+
break
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if !bound {
|
|
225
|
+
return Run{}, fmt.Errorf("%w: workflow %q no longer targets repo %q", ErrRestartConflict, wf.Name, previous.Repo)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
logicalID := previous.LogicalID
|
|
229
|
+
if logicalID == "" {
|
|
230
|
+
logicalID = identity.NewRunID(previous.Repo, previous.Workflow, previous.Ticket.Key)
|
|
231
|
+
}
|
|
232
|
+
// The lifecycle gate makes max+1 allocation single-writer within the
|
|
233
|
+
// server. Because every prior attempt is persisted in relay_runs, the
|
|
234
|
+
// number remains stable across process restarts and repeated commands.
|
|
235
|
+
attempts, err := m.Runs.ListRuns(ctx, Filter{Repo: previous.Repo, Workflow: previous.Workflow, Ticket: ticket})
|
|
236
|
+
if err != nil {
|
|
237
|
+
return Run{}, fmt.Errorf("allocate restart attempt for %s: %w", ticket, err)
|
|
238
|
+
}
|
|
239
|
+
var attemptID AttemptID = 1
|
|
240
|
+
for _, candidate := range attempts {
|
|
241
|
+
candidateAttempt := candidate.AttemptID
|
|
242
|
+
if candidateAttempt == 0 {
|
|
243
|
+
candidateAttempt = 1
|
|
244
|
+
}
|
|
245
|
+
if candidateAttempt >= attemptID {
|
|
246
|
+
if candidateAttempt == ^AttemptID(0) {
|
|
247
|
+
return Run{}, fmt.Errorf("%w: attempt number exhausted for %s", ErrRestartConflict, ticket)
|
|
248
|
+
}
|
|
249
|
+
attemptID = candidateAttempt + 1
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
executionID := identity.NewAttemptRunID(logicalID, attemptID)
|
|
253
|
+
start := Start{
|
|
254
|
+
ID: executionID, LogicalID: logicalID, AttemptID: attemptID,
|
|
255
|
+
Repo: previous.Repo, RepoPath: rp.Path, Workflow: *wf, Ticket: previous.Ticket,
|
|
256
|
+
}
|
|
257
|
+
if start.Ticket.Key == "" {
|
|
258
|
+
start.Ticket.Key = ticket
|
|
259
|
+
}
|
|
260
|
+
if err := m.ensure(ctx, start); err != nil {
|
|
261
|
+
return Run{}, err
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// The real projection is inserted before the durable instance is started.
|
|
265
|
+
// A fake or an engine that cannot read it yet still gets a useful command
|
|
266
|
+
// result; a later poll/EnsureRun repairs a missing workflow instance.
|
|
267
|
+
if current, err := m.Runs.GetRun(ctx, executionID); err == nil {
|
|
268
|
+
return current, nil
|
|
269
|
+
}
|
|
270
|
+
now := time.Now().UTC()
|
|
271
|
+
return Run{
|
|
272
|
+
ID: executionID, LogicalID: logicalID, AttemptID: attemptID,
|
|
273
|
+
Repo: start.Repo, Workflow: start.Workflow.Name, Ticket: start.Ticket,
|
|
274
|
+
State: StateStarting, StartedAt: now, UpdatedAt: now,
|
|
275
|
+
}, nil
|
|
276
|
+
}
|
|
277
|
+
|
|
110
278
|
// CancelByTicket resolves the active run through FindRunByTicket, then
|
|
111
279
|
// calls Executor.CancelRun.
|
|
112
280
|
func (m *RunManager) CancelByTicket(ctx context.Context, ticket, reason string) error {
|
package/internal/run/run.go
CHANGED
|
@@ -4,6 +4,7 @@ package run
|
|
|
4
4
|
|
|
5
5
|
import (
|
|
6
6
|
"context"
|
|
7
|
+
"errors"
|
|
7
8
|
"time"
|
|
8
9
|
|
|
9
10
|
"github.com/rajpopat27/relay-flow/internal/config"
|
|
@@ -13,6 +14,7 @@ import (
|
|
|
13
14
|
)
|
|
14
15
|
|
|
15
16
|
type ID = identity.RunID
|
|
17
|
+
type AttemptID = identity.AttemptID
|
|
16
18
|
type NodeVisitID = identity.NodeVisitID
|
|
17
19
|
|
|
18
20
|
type State string
|
|
@@ -30,12 +32,20 @@ const (
|
|
|
30
32
|
// Start carries the immutable value snapshot of the accepted workflow for
|
|
31
33
|
// deterministic replay. The interpreter consumes only this snapshot.
|
|
32
34
|
type Start struct {
|
|
33
|
-
ID
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
// ID is the durable execution-attempt ID. The first attempt (attempt 1)
|
|
36
|
+
// uses the deterministic logical ID; explicit restarts use a fenced ID.
|
|
37
|
+
ID ID `json:"id"`
|
|
38
|
+
// LogicalID remains stable across explicit attempts and is used for
|
|
39
|
+
// task-system cancellation fencing and ticket lookup.
|
|
40
|
+
LogicalID ID `json:"logicalRunId,omitempty"`
|
|
41
|
+
// AttemptID is 1 for the original execution and increases for explicit
|
|
42
|
+
// restarts. Zero is accepted only for legacy callers and normalized to 1.
|
|
43
|
+
AttemptID AttemptID `json:"attemptId,omitempty"`
|
|
44
|
+
Repo string `json:"repo"`
|
|
45
|
+
RepoPath string `json:"repoPath"`
|
|
46
|
+
Workflow workflow.Workflow `json:"workflow"`
|
|
47
|
+
Ticket task.TicketRef `json:"ticket"`
|
|
48
|
+
Runtime RuntimePolicy `json:"runtime"`
|
|
39
49
|
}
|
|
40
50
|
|
|
41
51
|
type RuntimePolicy struct {
|
|
@@ -45,6 +55,8 @@ type RuntimePolicy struct {
|
|
|
45
55
|
|
|
46
56
|
type Work struct {
|
|
47
57
|
RunID ID
|
|
58
|
+
LogicalID ID
|
|
59
|
+
AttemptID AttemptID
|
|
48
60
|
Repo string
|
|
49
61
|
Workflow string
|
|
50
62
|
Parent task.TicketRef
|
|
@@ -81,6 +93,8 @@ type RetryStatus struct {
|
|
|
81
93
|
|
|
82
94
|
type Run struct {
|
|
83
95
|
ID ID `json:"id"`
|
|
96
|
+
LogicalID ID `json:"logicalRunId,omitempty"`
|
|
97
|
+
AttemptID AttemptID `json:"attemptId,omitempty"`
|
|
84
98
|
Repo string `json:"repo"`
|
|
85
99
|
Workflow string `json:"workflow"`
|
|
86
100
|
Ticket task.TicketRef `json:"ticket"`
|
|
@@ -106,6 +120,10 @@ type ReportAck struct {
|
|
|
106
120
|
Duplicate bool `json:"duplicate"`
|
|
107
121
|
}
|
|
108
122
|
|
|
123
|
+
// ErrRestartConflict means the ticket cannot accept an explicit restart in
|
|
124
|
+
// its current durable state. Server handlers map it to HTTP 409.
|
|
125
|
+
var ErrRestartConflict = errors.New("restart conflict")
|
|
126
|
+
|
|
109
127
|
// NodeRuntimeRegistration binds the OpenCode session emitted for one run/node.
|
|
110
128
|
type NodeRuntimeRegistration struct {
|
|
111
129
|
RunID ID `json:"runId"`
|
|
@@ -259,6 +259,106 @@ func TestDeterministicRunID(t *testing.T) {
|
|
|
259
259
|
}
|
|
260
260
|
}
|
|
261
261
|
|
|
262
|
+
func TestRestartByTicketCreatesNumericFreshAttempt(t *testing.T) {
|
|
263
|
+
log := newEventLog()
|
|
264
|
+
sys := &recordingSystem{log: log}
|
|
265
|
+
wf := testWorkflow("basicFlow")
|
|
266
|
+
latestWorkflow := testWorkflow("basicFlow")
|
|
267
|
+
latestNode := latestWorkflow.Nodes["coding"]
|
|
268
|
+
latestNode.Description = "latest workflow definition"
|
|
269
|
+
latestWorkflow.Nodes["coding"] = latestNode
|
|
270
|
+
logical := identity.NewRunID("payments", wf.Name, "PAY-101")
|
|
271
|
+
previous := run.Run{
|
|
272
|
+
ID: logical, LogicalID: logical, AttemptID: 1,
|
|
273
|
+
Repo: "payments", Workflow: wf.Name,
|
|
274
|
+
Ticket: task.TicketRef{ID: "1", Key: "PAY-101"}, State: run.StateCanceled,
|
|
275
|
+
}
|
|
276
|
+
queries := &fakeQueries{
|
|
277
|
+
byTicket: map[string]run.Run{"PAY-101": previous},
|
|
278
|
+
list: []run.Run{previous},
|
|
279
|
+
}
|
|
280
|
+
exec := &fakeExecutor{log: log, created: true}
|
|
281
|
+
repos := repo.NewRegistry()
|
|
282
|
+
repos.Replace(testRepo(sys))
|
|
283
|
+
workflows := &workflow.Registry{}
|
|
284
|
+
workflows.Replace(latestWorkflow)
|
|
285
|
+
m := &run.RunManager{Executor: exec, Runs: queries, Repos: repos, Workflows: workflows}
|
|
286
|
+
|
|
287
|
+
got, err := m.RestartByTicket(context.Background(), "PAY-101")
|
|
288
|
+
if err != nil {
|
|
289
|
+
t.Fatalf("RestartByTicket failed: %v", err)
|
|
290
|
+
}
|
|
291
|
+
wantID := identity.NewAttemptRunID(logical, 2)
|
|
292
|
+
if got.ID != wantID || got.LogicalID != logical || got.AttemptID != 2 {
|
|
293
|
+
t.Fatalf("restart run = %+v, want ID=%q logical=%q attempt=2", got, wantID, logical)
|
|
294
|
+
}
|
|
295
|
+
if len(exec.ensures) != 1 {
|
|
296
|
+
t.Fatalf("EnsureRun calls = %d, want 1", len(exec.ensures))
|
|
297
|
+
}
|
|
298
|
+
start := exec.ensures[0]
|
|
299
|
+
if start.ID != wantID || start.AttemptID != 2 || start.LogicalID != logical {
|
|
300
|
+
t.Fatalf("restart start = %+v, want numeric attempt 2 and fenced ID", start)
|
|
301
|
+
}
|
|
302
|
+
if start.Workflow.Name != wf.Name || start.Workflow.Nodes["coding"].Description != "latest workflow definition" {
|
|
303
|
+
t.Fatalf("restart did not use latest workflow snapshot: %+v", start.Workflow)
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
func TestRestartByTicketIsIdempotentForActiveFreshAttempt(t *testing.T) {
|
|
308
|
+
attempt := run.Run{
|
|
309
|
+
ID: "payments/basicFlow/PAY-101~attempt~2", LogicalID: "payments/basicFlow/PAY-101", AttemptID: 2,
|
|
310
|
+
Repo: "payments", Workflow: "basicFlow", Ticket: task.TicketRef{Key: "PAY-101"}, State: run.StateBlocked,
|
|
311
|
+
}
|
|
312
|
+
queries := &fakeQueries{byTicket: map[string]run.Run{"PAY-101": attempt}}
|
|
313
|
+
exec := &fakeExecutor{log: newEventLog()}
|
|
314
|
+
m := &run.RunManager{Executor: exec, Runs: queries}
|
|
315
|
+
got, err := m.RestartByTicket(context.Background(), "PAY-101")
|
|
316
|
+
if err != nil {
|
|
317
|
+
t.Fatalf("idempotent restart failed: %v", err)
|
|
318
|
+
}
|
|
319
|
+
if got.ID != attempt.ID || len(exec.ensures) != 0 {
|
|
320
|
+
t.Fatalf("idempotent restart = %+v, EnsureRun calls=%d", got, len(exec.ensures))
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
func TestRestartByTicketWaitsForCancelingAttempt(t *testing.T) {
|
|
325
|
+
attempt := run.Run{
|
|
326
|
+
ID: "payments/basicFlow/PAY-101", LogicalID: "payments/basicFlow/PAY-101", AttemptID: 1,
|
|
327
|
+
Repo: "payments", Workflow: "basicFlow", Ticket: task.TicketRef{Key: "PAY-101"}, State: run.StateCanceling,
|
|
328
|
+
}
|
|
329
|
+
queries := &fakeQueries{byTicket: map[string]run.Run{"PAY-101": attempt}}
|
|
330
|
+
m := &run.RunManager{Executor: &fakeExecutor{log: newEventLog()}, Runs: queries}
|
|
331
|
+
_, err := m.RestartByTicket(context.Background(), "PAY-101")
|
|
332
|
+
if !errors.Is(err, run.ErrRestartConflict) {
|
|
333
|
+
t.Fatalf("restart error = %v, want ErrRestartConflict", err)
|
|
334
|
+
}
|
|
335
|
+
if !strings.Contains(err.Error(), "wait for cancellation") {
|
|
336
|
+
t.Fatalf("restart error = %v, want actionable cancellation guidance", err)
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
func TestEnsureRunReusesActiveRestartAttempt(t *testing.T) {
|
|
341
|
+
log := newEventLog()
|
|
342
|
+
sys := &recordingSystem{log: log, hasCancel: true}
|
|
343
|
+
logical := identity.NewRunID("payments", "basicFlow", "PAY-101")
|
|
344
|
+
attempt := run.Run{
|
|
345
|
+
ID: identity.NewAttemptRunID(logical, 2), LogicalID: logical, AttemptID: 2,
|
|
346
|
+
Repo: "payments", Workflow: "basicFlow", Ticket: task.TicketRef{Key: "PAY-101"}, State: run.StateBlocked,
|
|
347
|
+
}
|
|
348
|
+
exec := &fakeExecutor{log: log}
|
|
349
|
+
m := &run.RunManager{Executor: exec, Runs: &fakeQueries{list: []run.Run{attempt}}}
|
|
350
|
+
ticket := task.Ticket{ID: "1", Key: "PAY-101", WorkflowClaims: []string{"wf:basicFlow"}}
|
|
351
|
+
if err := m.EnsureRun(context.Background(), testRepo(sys), testWorkflow("basicFlow"), ticket); err != nil {
|
|
352
|
+
t.Fatal(err)
|
|
353
|
+
}
|
|
354
|
+
if len(exec.ensures) != 1 || exec.ensures[0].ID != attempt.ID {
|
|
355
|
+
t.Fatalf("ensures = %+v, want active restart ID %q", exec.ensures, attempt.ID)
|
|
356
|
+
}
|
|
357
|
+
if sys.hasCommentN != 0 {
|
|
358
|
+
t.Fatal("active restart attempt checked cancellation marker")
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
262
362
|
func TestCancelByTicketResolvesActiveRun(t *testing.T) {
|
|
263
363
|
rid := identity.NewRunID("payments", "basicFlow", "PAY-101")
|
|
264
364
|
queries := &fakeQueries{byTicket: map[string]run.Run{
|
|
@@ -3,8 +3,10 @@ package server_test
|
|
|
3
3
|
import (
|
|
4
4
|
"bytes"
|
|
5
5
|
"encoding/json"
|
|
6
|
+
"fmt"
|
|
6
7
|
"io"
|
|
7
8
|
"net/http"
|
|
9
|
+
"strings"
|
|
8
10
|
"testing"
|
|
9
11
|
"time"
|
|
10
12
|
|
|
@@ -211,6 +213,58 @@ func TestRepoAndRunEndpoints(t *testing.T) {
|
|
|
211
213
|
}
|
|
212
214
|
}
|
|
213
215
|
|
|
216
|
+
func TestRunRestartEndpointReturnsFreshAttempt(t *testing.T) {
|
|
217
|
+
want := run.Run{
|
|
218
|
+
ID: "payments/basicFlow/PAY-101~attempt~2", LogicalID: "payments/basicFlow/PAY-101", AttemptID: 2,
|
|
219
|
+
Repo: "payments", Workflow: "basicFlow", Ticket: task.TicketRef{Key: "PAY-101"}, State: run.StateStarting,
|
|
220
|
+
}
|
|
221
|
+
fake := &fakeServices{restartRun: want}
|
|
222
|
+
c, cleanup := startHandler(t, fake)
|
|
223
|
+
defer cleanup()
|
|
224
|
+
|
|
225
|
+
code, env := do(t, c, http.MethodPost, "http://relay/runs/by-ticket/PAY-101/restart", nil)
|
|
226
|
+
if code != http.StatusOK || !env.OK {
|
|
227
|
+
t.Fatalf("POST restart: code=%d env=%+v", code, env)
|
|
228
|
+
}
|
|
229
|
+
if !bytes.Contains(env.Data, []byte(`"attemptId":2`)) ||
|
|
230
|
+
!bytes.Contains(env.Data, []byte(`"logicalRunId":"payments/basicFlow/PAY-101"`)) {
|
|
231
|
+
t.Fatalf("restart response omitted attempt identity: %s", env.Data)
|
|
232
|
+
}
|
|
233
|
+
if len(fake.restarts) != 1 || fake.restarts[0] != "PAY-101" {
|
|
234
|
+
t.Fatalf("restart calls = %v", fake.restarts)
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
func TestRunRestartConflictMapsTo409(t *testing.T) {
|
|
239
|
+
fake := &fakeServices{restartErr: fmt.Errorf("%w: ticket is still canceling", run.ErrRestartConflict)}
|
|
240
|
+
c, cleanup := startHandler(t, fake)
|
|
241
|
+
defer cleanup()
|
|
242
|
+
code, env := do(t, c, http.MethodPost, "http://relay/runs/by-ticket/PAY-101/restart", nil)
|
|
243
|
+
if code != http.StatusConflict || env.OK || env.Error == nil || !strings.Contains(env.Error.Message, "ticket is still canceling") {
|
|
244
|
+
t.Fatalf("restart conflict: code=%d env=%+v", code, env)
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
func TestBlockedRunGetShowsStatusAction(t *testing.T) {
|
|
249
|
+
blocked := run.Run{
|
|
250
|
+
ID: "payments/basicFlow/PAY-101~attempt~2", LogicalID: "payments/basicFlow/PAY-101", AttemptID: 2,
|
|
251
|
+
State: run.StateBlocked, CurrentNode: "start",
|
|
252
|
+
Ticket: task.TicketRef{Key: "PAY-101"},
|
|
253
|
+
LastError: "Human-owned ticket status \"Blocked\" conflicts with the workflow start transition. Move ticket PAY-101 to an allowed active start status; relay-flow will retry automatically",
|
|
254
|
+
}
|
|
255
|
+
c, cleanup := startHandler(t, &fakeServices{runs: []run.Run{blocked}})
|
|
256
|
+
defer cleanup()
|
|
257
|
+
code, env := do(t, c, http.MethodGet, "http://relay/runs/by-ticket/PAY-101", nil)
|
|
258
|
+
if code != http.StatusOK || !env.OK {
|
|
259
|
+
t.Fatalf("GET blocked run: code=%d env=%+v", code, env)
|
|
260
|
+
}
|
|
261
|
+
for _, want := range []string{`"state":"blocked"`, `"currentNode":"start"`, `allowed active start status`, `automatically`} {
|
|
262
|
+
if !bytes.Contains(env.Data, []byte(want)) {
|
|
263
|
+
t.Fatalf("blocked run response missing %q: %s", want, env.Data)
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
214
268
|
func TestRunEndpointsExposeRetryDetails(t *testing.T) {
|
|
215
269
|
next := time.Now().UTC().Add(time.Minute)
|
|
216
270
|
retrying := run.Run{
|
|
@@ -205,6 +205,17 @@ func (c *Client) RegisterNodeSession(ctx context.Context, registration run.NodeR
|
|
|
205
205
|
return ack, nil
|
|
206
206
|
}
|
|
207
207
|
|
|
208
|
+
// RestartRun creates or returns the active fresh attempt for a canceled
|
|
209
|
+
// ticket. The server resolves the current repo/workflow and task-system
|
|
210
|
+
// boundaries; the client only transports the command.
|
|
211
|
+
func (c *Client) RestartRun(ctx context.Context, ticket string) (run.Run, error) {
|
|
212
|
+
var out run.Run
|
|
213
|
+
if err := c.call(ctx, http.MethodPost, "/runs/by-ticket/"+url.PathEscape(ticket)+"/restart", nil, &out); err != nil {
|
|
214
|
+
return run.Run{}, err
|
|
215
|
+
}
|
|
216
|
+
return out, nil
|
|
217
|
+
}
|
|
218
|
+
|
|
208
219
|
// CancelRun cancels the active run for the given ticket with a reason.
|
|
209
220
|
func (c *Client) CancelRun(ctx context.Context, ticket, reason string) error {
|
|
210
221
|
payload, _ := json.Marshal(map[string]string{"reason": reason})
|
|
@@ -34,6 +34,9 @@ type fakeServices struct {
|
|
|
34
34
|
runtimeAck run.NodeRuntimeRegistrationAck
|
|
35
35
|
processedReports map[string]bool
|
|
36
36
|
submittedReports int
|
|
37
|
+
restartRun run.Run
|
|
38
|
+
restartErr error
|
|
39
|
+
restarts []string
|
|
37
40
|
}
|
|
38
41
|
|
|
39
42
|
func (f *fakeServices) SubmitWorkflow(_ context.Context, yaml []byte) (*workflow.Workflow, error) {
|
|
@@ -79,6 +82,21 @@ func (f *fakeServices) GetRunByTicket(_ context.Context, ticket string) (run.Run
|
|
|
79
82
|
}
|
|
80
83
|
return run.Run{}, errNotFound{ticket}
|
|
81
84
|
}
|
|
85
|
+
func (f *fakeServices) RestartRun(_ context.Context, ticket string) (run.Run, error) {
|
|
86
|
+
f.restarts = append(f.restarts, ticket)
|
|
87
|
+
if f.restartErr != nil {
|
|
88
|
+
return run.Run{}, f.restartErr
|
|
89
|
+
}
|
|
90
|
+
if f.restartRun.ID != "" {
|
|
91
|
+
return f.restartRun, nil
|
|
92
|
+
}
|
|
93
|
+
for _, r := range f.runs {
|
|
94
|
+
if r.Ticket.Key == ticket {
|
|
95
|
+
return r, nil
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return run.Run{}, errNotFound{ticket}
|
|
99
|
+
}
|
|
82
100
|
func (f *fakeServices) CancelRun(_ context.Context, ticket, _ string) error {
|
|
83
101
|
for i, r := range f.runs {
|
|
84
102
|
if r.Ticket.Key == ticket {
|
|
@@ -31,6 +31,7 @@ type Deps interface {
|
|
|
31
31
|
// Runs
|
|
32
32
|
ListRuns(ctx context.Context, filter run.Filter) ([]run.Run, error)
|
|
33
33
|
GetRunByTicket(ctx context.Context, ticket string) (run.Run, error)
|
|
34
|
+
RestartRun(ctx context.Context, ticket string) (run.Run, error)
|
|
34
35
|
CancelRun(ctx context.Context, ticket, reason string) error
|
|
35
36
|
|
|
36
37
|
// Reports
|
|
@@ -117,6 +118,8 @@ func writeEnv(w http.ResponseWriter, status int, env envelope) {
|
|
|
117
118
|
// anything else is an unexpected 500.
|
|
118
119
|
func mapErr(w http.ResponseWriter, err error) {
|
|
119
120
|
switch {
|
|
121
|
+
case errors.Is(err, run.ErrRestartConflict):
|
|
122
|
+
writeErr(w, http.StatusConflict, "conflict", err.Error())
|
|
120
123
|
case errors.Is(err, ErrNotFound):
|
|
121
124
|
writeErr(w, http.StatusNotFound, "notFound", err.Error())
|
|
122
125
|
case errors.Is(err, ErrConflict):
|
|
@@ -475,6 +478,7 @@ func (s *server) handleRuns(w http.ResponseWriter, r *http.Request) {
|
|
|
475
478
|
|
|
476
479
|
func (s *server) handleRunByTicket(w http.ResponseWriter, r *http.Request) {
|
|
477
480
|
// /runs/by-ticket/{key} GET
|
|
481
|
+
// /runs/by-ticket/{key}/restart POST
|
|
478
482
|
// /runs/by-ticket/{key}/cancel POST
|
|
479
483
|
rest := strings.TrimPrefix(r.URL.Path, "/runs/by-ticket/")
|
|
480
484
|
parts := strings.Split(rest, "/")
|
|
@@ -490,6 +494,18 @@ func (s *server) handleRunByTicket(w http.ResponseWriter, r *http.Request) {
|
|
|
490
494
|
writeOK(w, http.StatusOK, rn)
|
|
491
495
|
return
|
|
492
496
|
}
|
|
497
|
+
if len(parts) == 2 && parts[0] != "" && parts[1] == "restart" {
|
|
498
|
+
if !methodOnly(w, r, http.MethodPost) {
|
|
499
|
+
return
|
|
500
|
+
}
|
|
501
|
+
rn, err := s.deps.RestartRun(r.Context(), parts[0])
|
|
502
|
+
if err != nil {
|
|
503
|
+
mapErr(w, err)
|
|
504
|
+
return
|
|
505
|
+
}
|
|
506
|
+
writeOK(w, http.StatusOK, rn)
|
|
507
|
+
return
|
|
508
|
+
}
|
|
493
509
|
if len(parts) == 2 && parts[0] != "" && parts[1] == "cancel" {
|
|
494
510
|
if !methodOnly(w, r, http.MethodPost) {
|
|
495
511
|
return
|
|
@@ -809,6 +809,21 @@ func (s *system) lifecycleDefaults(builtin config.RawValues) config.RawValues {
|
|
|
809
809
|
return config.Merge(builtin, inherited)
|
|
810
810
|
}
|
|
811
811
|
|
|
812
|
+
// PrepareRestart reopens relay-owned mailbox state for a new explicit
|
|
813
|
+
// attempt. The parent is deliberately not changed here; the start task
|
|
814
|
+
// configuration checks the parent's current state and leaves a human-owned
|
|
815
|
+
// Blocked/Deferred/Closed status untouched. Mailbox states outside the
|
|
816
|
+
// relay-owned open/in_progress/closed set return a conflict.
|
|
817
|
+
func (s *system) PrepareRestart(ctx context.Context, _ task.TicketRef, mailboxes []task.Mailbox) error {
|
|
818
|
+
for _, mailbox := range mailboxes {
|
|
819
|
+
if err := s.reconcileIssue(ctx, mailbox.Key,
|
|
820
|
+
[]string{statusOpen, statusInProgress, statusClosed}, statusOpen, ""); err != nil {
|
|
821
|
+
return fmt.Errorf("reopen mailbox %s for restart: %w", mailbox.Key, err)
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
return nil
|
|
825
|
+
}
|
|
826
|
+
|
|
812
827
|
// ResetForRecovery reopens the parent and every known mailbox, clearing any
|
|
813
828
|
// deferred state while preserving comments, labels, descriptions, history,
|
|
814
829
|
// and issues themselves.
|
|
@@ -837,4 +852,5 @@ func (s *system) ResetForRecovery(ctx context.Context, parent task.TicketRef, ma
|
|
|
837
852
|
var (
|
|
838
853
|
_ task.System = (*system)(nil)
|
|
839
854
|
_ task.LifecycleDefaults = (*system)(nil)
|
|
855
|
+
_ task.RestartPreparer = (*system)(nil)
|
|
840
856
|
)
|