relay-flow 0.3.8-alpha → 0.3.9-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/internal/execution/goworkflows/activities.go +11 -0
- package/internal/execution/goworkflows/cancellation_test.go +50 -0
- package/internal/execution/goworkflows/engine.go +278 -9
- package/internal/execution/goworkflows/fakes_test.go +10 -1
- package/internal/execution/goworkflows/interpreter.go +17 -1
- package/internal/execution/goworkflows/projection.go +8 -0
- package/internal/execution/goworkflows/recovery_test.go +482 -0
- package/internal/execution/projection/detail_test.go +38 -0
- package/internal/execution/projection/projection.go +80 -15
- package/internal/harness/opencode/opencode_test.go +1 -1
- package/internal/harness/opencode/repo_setup.go +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -24,13 +24,13 @@ same `relay-flow-plugin` package with host-specific entrypoints.
|
|
|
24
24
|
**OpenCode**
|
|
25
25
|
|
|
26
26
|
```sh
|
|
27
|
-
opencode plugin relay-flow-plugin@0.3.
|
|
27
|
+
opencode plugin relay-flow-plugin@0.3.9-alpha
|
|
28
28
|
```
|
|
29
29
|
|
|
30
30
|
**Pi**
|
|
31
31
|
|
|
32
32
|
```sh
|
|
33
|
-
pi install npm:relay-flow-plugin@0.3.
|
|
33
|
+
pi install npm:relay-flow-plugin@0.3.9-alpha
|
|
34
34
|
```
|
|
35
35
|
|
|
36
36
|
Pi loads the package's `pi.ts` extension from its manifest. Do not add
|
|
@@ -160,7 +160,7 @@ OpenCode plugin configuration uses both entrypoints. The server entrypoint is li
|
|
|
160
160
|
```json
|
|
161
161
|
{
|
|
162
162
|
"$schema": "https://opencode.ai/config.json",
|
|
163
|
-
"plugin": ["relay-flow-plugin@0.3.
|
|
163
|
+
"plugin": ["relay-flow-plugin@0.3.9-alpha"]
|
|
164
164
|
}
|
|
165
165
|
```
|
|
166
166
|
|
|
@@ -169,7 +169,7 @@ The native HITL approval entrypoint is listed in `.opencode/tui.json`:
|
|
|
169
169
|
```json
|
|
170
170
|
{
|
|
171
171
|
"$schema": "https://opencode.ai/tui.json",
|
|
172
|
-
"plugin": ["relay-flow-plugin@0.3.
|
|
172
|
+
"plugin": ["relay-flow-plugin@0.3.9-alpha"]
|
|
173
173
|
}
|
|
174
174
|
```
|
|
175
175
|
|
|
@@ -185,7 +185,7 @@ Pi plugin: install the same published package manually in Pi's global package
|
|
|
185
185
|
settings before starting a Pi harness session:
|
|
186
186
|
|
|
187
187
|
```sh
|
|
188
|
-
pi install npm:relay-flow-plugin@0.3.
|
|
188
|
+
pi install npm:relay-flow-plugin@0.3.9-alpha
|
|
189
189
|
```
|
|
190
190
|
|
|
191
191
|
Relay-flow does not install or configure the package automatically. Pi resolves
|
|
@@ -161,6 +161,17 @@ func (a *Activities) LoadNodeRuntime(ctx context.Context, id run.ID, node string
|
|
|
161
161
|
return a.Runs.loadNodeRuntime(ctx, id, node)
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
+
// LoadCancellationReason reads the operator reason persisted by the
|
|
165
|
+
// cancellation CAS. It is an activity because workflow code cannot access
|
|
166
|
+
// the relay projection directly.
|
|
167
|
+
func (a *Activities) LoadCancellationReason(ctx context.Context, id run.ID) (string, error) {
|
|
168
|
+
r, err := a.Runs.get(ctx, id)
|
|
169
|
+
if err != nil {
|
|
170
|
+
return "", err
|
|
171
|
+
}
|
|
172
|
+
return r.LastError, nil
|
|
173
|
+
}
|
|
174
|
+
|
|
164
175
|
// EnsureNodeRuntime uses only persisted terminal/session IDs on the normal
|
|
165
176
|
// path. A live terminal is rebound to the new visit; otherwise EnsureTerminal
|
|
166
177
|
// creates a replacement and its direct ID is persisted immediately. A stored
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
package goworkflows
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"database/sql"
|
|
6
|
+
"errors"
|
|
7
|
+
"fmt"
|
|
8
|
+
"testing"
|
|
9
|
+
"time"
|
|
10
|
+
|
|
11
|
+
"github.com/cschleiden/go-workflows/backend"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
func TestDetachedContextPreservesDeadline(t *testing.T) {
|
|
15
|
+
deadline := time.Now().Add(2 * time.Second)
|
|
16
|
+
parent, cancel := context.WithDeadline(context.Background(), deadline)
|
|
17
|
+
defer cancel()
|
|
18
|
+
detached, cleanup := detachedContext(parent)
|
|
19
|
+
defer cleanup()
|
|
20
|
+
cancel()
|
|
21
|
+
if err := detached.Err(); err != nil {
|
|
22
|
+
t.Fatalf("detached context inherited parent cancellation: %v", err)
|
|
23
|
+
}
|
|
24
|
+
got, ok := detached.Deadline()
|
|
25
|
+
if !ok || got.After(deadline.Add(10*time.Millisecond)) || got.Before(deadline.Add(-10*time.Millisecond)) {
|
|
26
|
+
t.Fatalf("detached deadline = %v, want approximately %v", got, deadline)
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
func TestIsMissingWorkflowInstanceOnlyMatchesConfirmedAbsence(t *testing.T) {
|
|
31
|
+
tests := []struct {
|
|
32
|
+
name string
|
|
33
|
+
err error
|
|
34
|
+
want bool
|
|
35
|
+
}{
|
|
36
|
+
{name: "sql no rows", err: sql.ErrNoRows, want: true},
|
|
37
|
+
{name: "wrapped sql no rows", err: fmt.Errorf("lookup instance: %w", sql.ErrNoRows), want: true},
|
|
38
|
+
{name: "backend not found", err: backend.ErrInstanceNotFound, want: true},
|
|
39
|
+
{name: "wrapped backend not found", err: fmt.Errorf("cancel instance: %w", backend.ErrInstanceNotFound), want: true},
|
|
40
|
+
{name: "database locked", err: errors.New("database is locked"), want: false},
|
|
41
|
+
{name: "context canceled", err: errors.New("context canceled"), want: false},
|
|
42
|
+
}
|
|
43
|
+
for _, tt := range tests {
|
|
44
|
+
t.Run(tt.name, func(t *testing.T) {
|
|
45
|
+
if got := isMissingWorkflowInstance(tt.err); got != tt.want {
|
|
46
|
+
t.Fatalf("isMissingWorkflowInstance(%v) = %v, want %v", tt.err, got, tt.want)
|
|
47
|
+
}
|
|
48
|
+
})
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -27,8 +27,10 @@ import (
|
|
|
27
27
|
"github.com/rajpopat27/relay-flow/internal/harness"
|
|
28
28
|
"github.com/rajpopat27/relay-flow/internal/identity"
|
|
29
29
|
"github.com/rajpopat27/relay-flow/internal/repo"
|
|
30
|
+
"github.com/rajpopat27/relay-flow/internal/retry"
|
|
30
31
|
"github.com/rajpopat27/relay-flow/internal/run"
|
|
31
32
|
"github.com/rajpopat27/relay-flow/internal/runner"
|
|
33
|
+
"github.com/rajpopat27/relay-flow/internal/task"
|
|
32
34
|
"github.com/rajpopat27/relay-flow/internal/workflow"
|
|
33
35
|
)
|
|
34
36
|
|
|
@@ -61,6 +63,7 @@ type Engine struct {
|
|
|
61
63
|
runtime run.RuntimePolicy
|
|
62
64
|
|
|
63
65
|
mu sync.RWMutex
|
|
66
|
+
cancelMu sync.Mutex // serializes cancellation history check + request
|
|
64
67
|
snapshots map[run.ID]*workflow.Workflow // in-memory cache; history is authoritative
|
|
65
68
|
|
|
66
69
|
workerCtx context.Context
|
|
@@ -177,6 +180,11 @@ func (e *Engine) Start(ctx context.Context) error {
|
|
|
177
180
|
if err := e.actWorker.Start(e.workerCtx); err != nil {
|
|
178
181
|
return fmt.Errorf("start activity worker: %w", err)
|
|
179
182
|
}
|
|
183
|
+
// A cancellation request can outlive the engine instance that accepted it.
|
|
184
|
+
// Reconcile those projections before normal pollers start; a missing
|
|
185
|
+
// workflow instance is a confirmed terminal execution boundary, while
|
|
186
|
+
// other lookup failures remain retryable and are logged below.
|
|
187
|
+
e.reconcileCancelingRuns(ctx)
|
|
180
188
|
// Startup retention sweep (pre-poller window): remove old terminal
|
|
181
189
|
// projection rows and their engine histories; nonterminal runs stay.
|
|
182
190
|
cutoff := time.Now().Add(-e.retention)
|
|
@@ -202,6 +210,7 @@ func (e *Engine) registerActivities() error {
|
|
|
202
210
|
a.EnsureEnvironment,
|
|
203
211
|
a.SetEnvironmentStatus,
|
|
204
212
|
a.LoadNodeRuntime,
|
|
213
|
+
a.LoadCancellationReason,
|
|
205
214
|
a.EnsureNodeRuntime,
|
|
206
215
|
a.CloseTerminals,
|
|
207
216
|
a.CleanupRun,
|
|
@@ -462,26 +471,286 @@ func (e *Engine) workflowOf(ctx context.Context, id run.ID) (*workflow.Workflow,
|
|
|
462
471
|
return nil, fmt.Errorf("no workflow snapshot in history for run %s", id)
|
|
463
472
|
}
|
|
464
473
|
|
|
465
|
-
// CancelRun
|
|
466
|
-
//
|
|
474
|
+
// CancelRun requests cancellation of the durable workflow. The projection
|
|
475
|
+
// transition is compare-and-set: a concurrent completion wins if it reaches a
|
|
476
|
+
// terminal state first, while a persisted canceling state is the durable
|
|
477
|
+
// request that startup reconciliation retries.
|
|
467
478
|
func (e *Engine) CancelRun(ctx context.Context, id run.ID, reason string) error {
|
|
468
|
-
|
|
479
|
+
r, err := e.runs.beginCancellation(ctx, id, reason)
|
|
480
|
+
if err != nil {
|
|
469
481
|
return fmt.Errorf("resolve run %s: %w", id, err)
|
|
470
482
|
}
|
|
471
|
-
if
|
|
483
|
+
if r.State == run.StateCompleted || r.State == run.StateCanceled {
|
|
484
|
+
return nil
|
|
485
|
+
}
|
|
486
|
+
if r.State != run.StateCanceling {
|
|
487
|
+
return fmt.Errorf("cancel %s: state changed to %s", id, r.State)
|
|
488
|
+
}
|
|
489
|
+
// The reason persisted by the first successful CAS is authoritative for
|
|
490
|
+
// every later cancellation request.
|
|
491
|
+
return e.reconcileCancellation(ctx, r, r.LastError)
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// isMissingWorkflowInstance identifies only confirmed absence of the active
|
|
495
|
+
// go-workflows execution. Database and context failures remain retryable.
|
|
496
|
+
func isMissingWorkflowInstance(err error) bool {
|
|
497
|
+
return errors.Is(err, sql.ErrNoRows) || errors.Is(err, backend.ErrInstanceNotFound)
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// detachedContext preserves a caller deadline while detaching cancellation
|
|
501
|
+
// from the caller. Startup cleanup must not be able to outlive its bounded
|
|
502
|
+
// reconciliation window.
|
|
503
|
+
func detachedContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
|
504
|
+
base := context.WithoutCancel(ctx)
|
|
505
|
+
if deadline, ok := ctx.Deadline(); ok {
|
|
506
|
+
return context.WithDeadline(base, deadline)
|
|
507
|
+
}
|
|
508
|
+
return base, func() {}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
// lookupInstance returns the active execution when present. If only a finished
|
|
512
|
+
// row remains, finished is true; a missing row is the only absence outcome.
|
|
513
|
+
func (e *Engine) lookupInstance(ctx context.Context, id run.ID) (*goworkflow.Instance, bool, error) {
|
|
514
|
+
var execID string
|
|
515
|
+
var state int
|
|
516
|
+
err := e.db.QueryRowContext(ctx, `
|
|
517
|
+
SELECT execution_id, state FROM instances WHERE id = ?
|
|
518
|
+
ORDER BY CASE WHEN state = 0 THEN 0 ELSE 1 END, rowid DESC LIMIT 1`, string(id)).Scan(&execID, &state)
|
|
519
|
+
if err != nil {
|
|
520
|
+
return nil, false, fmt.Errorf("workflow instance %s not found: %w", id, err)
|
|
521
|
+
}
|
|
522
|
+
return &goworkflow.Instance{InstanceID: string(id), ExecutionID: execID}, state != 0, nil
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// reconcileCancellation retries the durable cancellation request while an
|
|
526
|
+
// execution is active and reconciles finished/missing executions separately.
|
|
527
|
+
func (e *Engine) reconcileCancellation(ctx context.Context, r run.Run, reason string) error {
|
|
528
|
+
inst, finished, err := e.lookupInstance(ctx, r.ID)
|
|
529
|
+
if err != nil {
|
|
530
|
+
if isMissingWorkflowInstance(err) {
|
|
531
|
+
return e.finalizeMissingCancellation(ctx, r, reason)
|
|
532
|
+
}
|
|
533
|
+
return fmt.Errorf("resolve workflow instance: %w", err)
|
|
534
|
+
}
|
|
535
|
+
if finished {
|
|
536
|
+
return e.reconcileFinishedCancellation(ctx, r, inst, reason)
|
|
537
|
+
}
|
|
538
|
+
if e.client == nil {
|
|
539
|
+
return errors.New("workflow engine is not started")
|
|
540
|
+
}
|
|
541
|
+
if err := e.requestCancellation(ctx, inst); err != nil {
|
|
542
|
+
if isMissingWorkflowInstance(err) {
|
|
543
|
+
return e.finalizeMissingCancellation(ctx, r, reason)
|
|
544
|
+
}
|
|
472
545
|
return err
|
|
473
546
|
}
|
|
474
|
-
|
|
547
|
+
return nil
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// requestCancellation checks durable history and appends at most one
|
|
551
|
+
// cancellation event for an execution. The mutex closes the check/request
|
|
552
|
+
// race between concurrent operator calls in this process; the relay lock
|
|
553
|
+
// prevents another relay-flow server from owning the same database.
|
|
554
|
+
func (e *Engine) requestCancellation(ctx context.Context, inst *goworkflow.Instance) error {
|
|
555
|
+
e.cancelMu.Lock()
|
|
556
|
+
defer e.cancelMu.Unlock()
|
|
557
|
+
var requested int
|
|
558
|
+
if err := e.db.QueryRowContext(ctx, `
|
|
559
|
+
SELECT EXISTS(
|
|
560
|
+
SELECT 1 FROM history
|
|
561
|
+
WHERE instance_id = ? AND execution_id = ? AND event_type = ?
|
|
562
|
+
UNION ALL
|
|
563
|
+
SELECT 1 FROM pending_events
|
|
564
|
+
WHERE instance_id = ? AND execution_id = ? AND event_type = ?
|
|
565
|
+
)`,
|
|
566
|
+
inst.InstanceID, inst.ExecutionID, history.EventType_WorkflowExecutionCanceled,
|
|
567
|
+
inst.InstanceID, inst.ExecutionID, history.EventType_WorkflowExecutionCanceled,
|
|
568
|
+
).Scan(&requested); err != nil {
|
|
569
|
+
return fmt.Errorf("inspect cancellation history: %w", err)
|
|
570
|
+
}
|
|
571
|
+
if requested != 0 {
|
|
572
|
+
return nil
|
|
573
|
+
}
|
|
574
|
+
return e.client.CancelWorkflowInstance(ctx, inst)
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// finalizeMissingCancellation performs the same roll-forward cleanup as the
|
|
578
|
+
// workflow's disconnected cancellation path when the workflow instance has
|
|
579
|
+
// genuinely disappeared. It intentionally does not recreate an execution.
|
|
580
|
+
func (e *Engine) finalizeMissingCancellation(ctx context.Context, r run.Run, reason string) error {
|
|
581
|
+
cleanupCtx, cleanupCancel := detachedContext(ctx)
|
|
582
|
+
defer cleanupCancel()
|
|
583
|
+
current, err := e.runs.get(cleanupCtx, r.ID)
|
|
584
|
+
if err != nil {
|
|
585
|
+
return fmt.Errorf("reconcile canceled run %s projection: %w", r.ID, err)
|
|
586
|
+
}
|
|
587
|
+
if current.State == run.StateCompleted || current.State == run.StateCanceled {
|
|
588
|
+
return nil
|
|
589
|
+
}
|
|
590
|
+
if current.State != run.StateCanceling {
|
|
591
|
+
return fmt.Errorf("reconcile canceled run %s: state changed to %s", r.ID, current.State)
|
|
592
|
+
}
|
|
593
|
+
if inst, finished, lookupErr := e.lookupInstance(cleanupCtx, r.ID); lookupErr == nil {
|
|
594
|
+
if finished {
|
|
595
|
+
return e.reconcileFinishedCancellation(cleanupCtx, current, inst, reason)
|
|
596
|
+
}
|
|
597
|
+
return fmt.Errorf("reconcile canceled run %s: workflow instance reappeared", r.ID)
|
|
598
|
+
} else if !isMissingWorkflowInstance(lookupErr) {
|
|
599
|
+
return fmt.Errorf("reconcile canceled run %s instance lookup: %w", r.ID, lookupErr)
|
|
600
|
+
}
|
|
601
|
+
return e.finalizeCancellationEffects(cleanupCtx, current, reason)
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// finalizeCancellationEffects is shared by missing and already-canceled
|
|
605
|
+
// engine executions. The final projection transition is conditional so a
|
|
606
|
+
// concurrent terminal projection cannot be overwritten.
|
|
607
|
+
func (e *Engine) finalizeCancellationEffects(ctx context.Context, r run.Run, reason string) error {
|
|
608
|
+
if r.State == run.StateCompleted || r.State == run.StateCanceled {
|
|
609
|
+
return nil
|
|
610
|
+
}
|
|
611
|
+
if r.State != run.StateCanceling {
|
|
612
|
+
return fmt.Errorf("finalize canceled run %s: state changed to %s", r.ID, r.State)
|
|
613
|
+
}
|
|
614
|
+
repoInfo, ok := e.activities.Repos.Get(r.Repo)
|
|
615
|
+
if !ok {
|
|
616
|
+
return fmt.Errorf("reconcile canceled run %s: repo %q is no longer registered", r.ID, r.Repo)
|
|
617
|
+
}
|
|
618
|
+
work := run.Work{
|
|
619
|
+
RunID: r.ID,
|
|
620
|
+
LogicalID: r.LogicalID,
|
|
621
|
+
AttemptID: r.AttemptID,
|
|
622
|
+
Repo: r.Repo,
|
|
623
|
+
Workflow: r.Workflow,
|
|
624
|
+
Parent: r.Ticket,
|
|
625
|
+
Runtime: e.runtime,
|
|
626
|
+
}
|
|
627
|
+
if reason == "" {
|
|
628
|
+
reason = "operator requested cancellation"
|
|
629
|
+
}
|
|
630
|
+
if err := e.activities.FinalizeNodeRuntimes(ctx, work, repoInfo.Path, e.runtime); err != nil {
|
|
631
|
+
return fmt.Errorf("finalize canceled run %s terminals: %w", r.ID, err)
|
|
632
|
+
}
|
|
633
|
+
markerID := r.LogicalID
|
|
634
|
+
if markerID == "" {
|
|
635
|
+
markerID = r.ID
|
|
636
|
+
}
|
|
637
|
+
if err := e.activities.Comment(ctx, r.Repo, run.CommentWork{
|
|
638
|
+
RunID: r.ID,
|
|
639
|
+
Item: task.Target{Parent: r.Ticket},
|
|
640
|
+
Body: "Run canceled: " + reason,
|
|
641
|
+
Marker: run.CancellationMarker(markerID),
|
|
642
|
+
}); err != nil {
|
|
643
|
+
return fmt.Errorf("finalize canceled run %s comment: %w", r.ID, err)
|
|
644
|
+
}
|
|
645
|
+
finished := time.Now().UTC()
|
|
646
|
+
updated, err := e.runs.updateStateIf(ctx, r.ID, run.StateCanceling, run.StateCanceled, "", &finished)
|
|
475
647
|
if err != nil {
|
|
476
|
-
return fmt.Errorf("
|
|
648
|
+
return fmt.Errorf("finalize canceled run %s projection: %w", r.ID, err)
|
|
477
649
|
}
|
|
478
|
-
if
|
|
479
|
-
|
|
650
|
+
if !updated {
|
|
651
|
+
latest, getErr := e.runs.get(ctx, r.ID)
|
|
652
|
+
if getErr == nil && (latest.State == run.StateCanceled || latest.State == run.StateCompleted) {
|
|
653
|
+
return nil
|
|
654
|
+
}
|
|
655
|
+
if getErr != nil {
|
|
656
|
+
return fmt.Errorf("finalize canceled run %s projection after race: %w", r.ID, getErr)
|
|
657
|
+
}
|
|
658
|
+
return fmt.Errorf("finalize canceled run %s: state changed to %s", r.ID, latest.State)
|
|
480
659
|
}
|
|
660
|
+
slog.Info("run canceled", "ticket", r.Ticket.Key, "runID", string(r.ID),
|
|
661
|
+
"repo", r.Repo, "workflow", r.Workflow, "state", run.StateCanceled)
|
|
481
662
|
return nil
|
|
482
663
|
}
|
|
483
664
|
|
|
484
|
-
//
|
|
665
|
+
// reconcileFinishedCancellation reads the terminal engine event before
|
|
666
|
+
// deciding whether a canceling projection should become completed or canceled.
|
|
667
|
+
func (e *Engine) reconcileFinishedCancellation(ctx context.Context, r run.Run, inst *goworkflow.Instance, reason string) error {
|
|
668
|
+
if e.backend == nil {
|
|
669
|
+
return errors.New("workflow backend is not started")
|
|
670
|
+
}
|
|
671
|
+
events, err := e.backend.GetWorkflowInstanceHistory(ctx, inst, nil)
|
|
672
|
+
if err != nil {
|
|
673
|
+
return fmt.Errorf("read finished workflow history: %w", err)
|
|
674
|
+
}
|
|
675
|
+
var finishedEvent *history.Event
|
|
676
|
+
canceledBeforeFinish := false
|
|
677
|
+
for _, event := range events {
|
|
678
|
+
switch event.Type {
|
|
679
|
+
case history.EventType_WorkflowExecutionCanceled:
|
|
680
|
+
// TicketWorkflow records this cancellation request first; after
|
|
681
|
+
// cancelCleanup returns, go-workflows may append a normal
|
|
682
|
+
// WorkflowExecutionFinished event. The earlier cancellation event
|
|
683
|
+
// remains authoritative for relay-flow semantics.
|
|
684
|
+
if finishedEvent == nil {
|
|
685
|
+
canceledBeforeFinish = true
|
|
686
|
+
}
|
|
687
|
+
case history.EventType_WorkflowExecutionFinished:
|
|
688
|
+
if finishedEvent == nil {
|
|
689
|
+
finishedEvent = event
|
|
690
|
+
}
|
|
691
|
+
case history.EventType_WorkflowExecutionTerminated:
|
|
692
|
+
return fmt.Errorf("workflow %s terminated without a cancellation or completion result", r.ID)
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
if canceledBeforeFinish {
|
|
696
|
+
return e.finalizeCancellationEffects(ctx, r, reason)
|
|
697
|
+
}
|
|
698
|
+
if finishedEvent == nil {
|
|
699
|
+
return fmt.Errorf("finished workflow %s has no terminal history event", r.ID)
|
|
700
|
+
}
|
|
701
|
+
{
|
|
702
|
+
finished := finishedEvent.Timestamp
|
|
703
|
+
updated, err := e.runs.updateStateIf(ctx, r.ID, run.StateCanceling, run.StateCompleted, "", &finished)
|
|
704
|
+
if err != nil {
|
|
705
|
+
return fmt.Errorf("reconcile completed run %s projection: %w", r.ID, err)
|
|
706
|
+
}
|
|
707
|
+
if !updated {
|
|
708
|
+
latest, getErr := e.runs.get(ctx, r.ID)
|
|
709
|
+
if getErr == nil && (latest.State == run.StateCompleted || latest.State == run.StateCanceled) {
|
|
710
|
+
return nil
|
|
711
|
+
}
|
|
712
|
+
if getErr != nil {
|
|
713
|
+
return fmt.Errorf("reconcile completed run %s projection after race: %w", r.ID, getErr)
|
|
714
|
+
}
|
|
715
|
+
return fmt.Errorf("reconcile completed run %s: state changed to %s", r.ID, latest.State)
|
|
716
|
+
}
|
|
717
|
+
return nil
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// reconcileCancelingRuns retries the durable cancellation request for every
|
|
722
|
+
// stale canceling projection. An active instance is not skipped: its cancel
|
|
723
|
+
// event is re-submitted until the workflow accepts it.
|
|
724
|
+
func (e *Engine) reconcileCancelingRuns(ctx context.Context) {
|
|
725
|
+
active := true
|
|
726
|
+
runs, err := e.runs.list(ctx, run.Filter{Active: &active})
|
|
727
|
+
if err != nil {
|
|
728
|
+
slog.Warn("reconcile canceling runs unavailable", "error", err)
|
|
729
|
+
return
|
|
730
|
+
}
|
|
731
|
+
for _, r := range runs {
|
|
732
|
+
if r.State != run.StateCanceling {
|
|
733
|
+
continue
|
|
734
|
+
}
|
|
735
|
+
if err := e.retryStartupCancellation(ctx, r); err != nil {
|
|
736
|
+
slog.Warn("reconcile canceling run failed", "runID", r.ID, "error", err)
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// retryStartupCancellation gives a durable canceling projection a bounded
|
|
742
|
+
// startup retry window. Normal polling deliberately does not recreate or
|
|
743
|
+
// advance canceling runs, so an active execution must receive its cancel
|
|
744
|
+
// event here or remain visibly retryable for the next restart.
|
|
745
|
+
func (e *Engine) retryStartupCancellation(ctx context.Context, r run.Run) error {
|
|
746
|
+
retryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
747
|
+
defer cancel()
|
|
748
|
+
return retry.Do(retryCtx, retry.DefaultBackoffPolicy, func() error {
|
|
749
|
+
return e.reconcileCancellation(retryCtx, r, r.LastError)
|
|
750
|
+
})
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
// instance resolves the current active execution for the durable run ID.
|
|
485
754
|
func (e *Engine) instance(ctx context.Context, id run.ID) (*goworkflow.Instance, error) {
|
|
486
755
|
var execID string
|
|
487
756
|
err := e.db.QueryRowContext(ctx,
|
|
@@ -226,7 +226,16 @@ func (s *fakeTaskSystem) Comment(_ context.Context, target task.Target, body, ma
|
|
|
226
226
|
if target.Mailbox != nil {
|
|
227
227
|
key = target.Mailbox.Key
|
|
228
228
|
}
|
|
229
|
-
|
|
229
|
+
s.mu.Lock()
|
|
230
|
+
for _, comment := range s.comments {
|
|
231
|
+
if comment.Key == key && comment.Marker == marker {
|
|
232
|
+
s.mu.Unlock()
|
|
233
|
+
return nil
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
fail := s.failComments
|
|
237
|
+
s.mu.Unlock()
|
|
238
|
+
if fail {
|
|
230
239
|
s.log.add("commentFail:" + key)
|
|
231
240
|
return errTransient
|
|
232
241
|
}
|
|
@@ -75,7 +75,23 @@ func (a *Activities) TicketWorkflow(ctx goworkflow.Context, start run.Start) err
|
|
|
75
75
|
WorkflowTaskConfig: start.Workflow.TaskConfig,
|
|
76
76
|
Runtime: start.Runtime,
|
|
77
77
|
}
|
|
78
|
-
|
|
78
|
+
// Cancellation events do not carry relay-flow's operator reason. Read
|
|
79
|
+
// the reason persisted by CancelRun through the durable disconnected
|
|
80
|
+
// retry path so cleanup cannot write a permanently wrong comment when
|
|
81
|
+
// the projection is temporarily unavailable.
|
|
82
|
+
dctx := goworkflow.NewDisconnectedContext(ctx)
|
|
83
|
+
reason, reasonErr := retryLoop(dctx, start.ID, a, work, "",
|
|
84
|
+
func(ctx2 goworkflow.Context) goworkflow.Future[string] {
|
|
85
|
+
return goworkflow.ExecuteActivity[string](ctx2, noNativeRetries,
|
|
86
|
+
a.LoadCancellationReason, start.ID)
|
|
87
|
+
})
|
|
88
|
+
if reasonErr != nil {
|
|
89
|
+
return reasonErr
|
|
90
|
+
}
|
|
91
|
+
if reason == "" {
|
|
92
|
+
reason = "canceled"
|
|
93
|
+
}
|
|
94
|
+
return a.cancelCleanup(ctx, work, start.RepoPath, reason)
|
|
79
95
|
}
|
|
80
96
|
return err
|
|
81
97
|
}
|
|
@@ -46,6 +46,14 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
|
|
|
46
46
|
return p.shared().UpdateState(ctx, id, state, lastErr, finished)
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
func (p *RunProjection) beginCancellation(ctx context.Context, id run.ID, reason string) (run.Run, error) {
|
|
50
|
+
return p.shared().BeginCancellation(ctx, id, reason)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
func (p *RunProjection) updateStateIf(ctx context.Context, id run.ID, expected, state run.State, lastErr string, finished *time.Time) (bool, error) {
|
|
54
|
+
return p.shared().UpdateStateIf(ctx, id, expected, state, lastErr, finished)
|
|
55
|
+
}
|
|
56
|
+
|
|
49
57
|
func (p *RunProjection) updateRetry(ctx context.Context, id run.ID, status *run.RetryStatus) error {
|
|
50
58
|
return p.shared().UpdateRetry(ctx, id, status)
|
|
51
59
|
}
|
|
@@ -9,6 +9,7 @@ import (
|
|
|
9
9
|
"testing"
|
|
10
10
|
"time"
|
|
11
11
|
|
|
12
|
+
"github.com/cschleiden/go-workflows/backend/history"
|
|
12
13
|
"github.com/rajpopat27/relay-flow/internal/execution/goworkflows"
|
|
13
14
|
"github.com/rajpopat27/relay-flow/internal/identity"
|
|
14
15
|
recoverpkg "github.com/rajpopat27/relay-flow/internal/recover"
|
|
@@ -200,14 +201,19 @@ func TestCancelRun(t *testing.T) {
|
|
|
200
201
|
// Exactly one parent cancellation comment with the stable marker.
|
|
201
202
|
wantMarker := string(rid) + ":cancellation"
|
|
202
203
|
var cancelComments int
|
|
204
|
+
var cancellationBody string
|
|
203
205
|
for _, c := range sys.commentBodies("PAY-101") {
|
|
204
206
|
if c.Marker == wantMarker {
|
|
205
207
|
cancelComments++
|
|
208
|
+
cancellationBody = c.Body
|
|
206
209
|
}
|
|
207
210
|
}
|
|
208
211
|
if cancelComments != 1 {
|
|
209
212
|
t.Fatalf("cancellation comments = %d, want 1 with marker %q", cancelComments, wantMarker)
|
|
210
213
|
}
|
|
214
|
+
if !strings.Contains(cancellationBody, "Run canceled: no longer needed") {
|
|
215
|
+
t.Fatalf("cancellation comment = %q, want persisted operator reason", cancellationBody)
|
|
216
|
+
}
|
|
211
217
|
|
|
212
218
|
// Mailbox statuses/history unchanged: the in-flight coding mailbox was
|
|
213
219
|
// not completed by cancellation.
|
|
@@ -220,6 +226,482 @@ func TestCancelRun(t *testing.T) {
|
|
|
220
226
|
}
|
|
221
227
|
}
|
|
222
228
|
|
|
229
|
+
func TestCancelRunFinalizesWhenWorkflowInstanceIsMissing(t *testing.T) {
|
|
230
|
+
log := newEventLog()
|
|
231
|
+
sys := newFakeTaskSystem(log)
|
|
232
|
+
fr := newFakeRunner(log)
|
|
233
|
+
deps := goworkflows.Dependencies{
|
|
234
|
+
Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
|
|
235
|
+
Runtime: &run.RuntimePolicy{},
|
|
236
|
+
}
|
|
237
|
+
dbPath := filepath.Join(t.TempDir(), "state.db")
|
|
238
|
+
first, err := goworkflows.New(dbPath, deps)
|
|
239
|
+
if err != nil {
|
|
240
|
+
t.Fatal(err)
|
|
241
|
+
}
|
|
242
|
+
if err := first.Start(context.Background()); err != nil {
|
|
243
|
+
t.Fatal(err)
|
|
244
|
+
}
|
|
245
|
+
rid, err := startRun(first, linearWorkflow(false))
|
|
246
|
+
if err != nil {
|
|
247
|
+
t.Fatal(err)
|
|
248
|
+
}
|
|
249
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
250
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
251
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
252
|
+
})
|
|
253
|
+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
254
|
+
if err := first.Shutdown(shutdownCtx); err != nil {
|
|
255
|
+
t.Fatal(err)
|
|
256
|
+
}
|
|
257
|
+
cancel()
|
|
258
|
+
|
|
259
|
+
// Simulate the cancellation crash boundary: the relay projection has a
|
|
260
|
+
// run, but the active go-workflows instance is gone.
|
|
261
|
+
db, err := sql.Open("sqlite", dbPath)
|
|
262
|
+
if err != nil {
|
|
263
|
+
t.Fatal(err)
|
|
264
|
+
}
|
|
265
|
+
if _, err := db.Exec(`DELETE FROM instances WHERE id = ?`, string(rid)); err != nil {
|
|
266
|
+
db.Close()
|
|
267
|
+
t.Fatal(err)
|
|
268
|
+
}
|
|
269
|
+
if err := db.Close(); err != nil {
|
|
270
|
+
t.Fatal(err)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
second, err := goworkflows.New(dbPath, deps)
|
|
274
|
+
if err != nil {
|
|
275
|
+
t.Fatal(err)
|
|
276
|
+
}
|
|
277
|
+
defer func() { _ = second.Shutdown(context.Background()) }()
|
|
278
|
+
sys.failComments = true
|
|
279
|
+
if err := second.CancelRun(context.Background(), rid, "operator canceled"); err == nil {
|
|
280
|
+
t.Fatal("CancelRun succeeded despite a cancellation-comment failure")
|
|
281
|
+
}
|
|
282
|
+
failed, err := second.GetRun(context.Background(), rid)
|
|
283
|
+
if err != nil {
|
|
284
|
+
t.Fatal(err)
|
|
285
|
+
}
|
|
286
|
+
if failed.State != run.StateCanceling || failed.LastError != "operator canceled" {
|
|
287
|
+
t.Fatalf("failed cancellation projection = %+v, want canceling with original reason", failed)
|
|
288
|
+
}
|
|
289
|
+
sys.failComments = false
|
|
290
|
+
if err := second.CancelRun(context.Background(), rid, "replacement reason"); err != nil {
|
|
291
|
+
t.Fatalf("CancelRun with missing workflow instance: %v", err)
|
|
292
|
+
}
|
|
293
|
+
got, err := second.GetRun(context.Background(), rid)
|
|
294
|
+
if err != nil {
|
|
295
|
+
t.Fatal(err)
|
|
296
|
+
}
|
|
297
|
+
if got.State != run.StateCanceled {
|
|
298
|
+
t.Fatalf("state = %q, want canceled", got.State)
|
|
299
|
+
}
|
|
300
|
+
active, err := second.HasActiveWorkflow(context.Background(), "basicFlow")
|
|
301
|
+
if err != nil {
|
|
302
|
+
t.Fatal(err)
|
|
303
|
+
}
|
|
304
|
+
if active {
|
|
305
|
+
t.Fatal("canceled run is still counted as an active workflow")
|
|
306
|
+
}
|
|
307
|
+
if fr.liveTerminals() != 0 {
|
|
308
|
+
t.Fatalf("missing-instance cancellation left %d live terminals", fr.liveTerminals())
|
|
309
|
+
}
|
|
310
|
+
comments := sys.commentBodies("PAY-101")
|
|
311
|
+
if len(comments) != 1 || comments[0].Marker != string(rid)+":cancellation" {
|
|
312
|
+
t.Fatalf("cancellation comments = %#v, want one stable cancellation marker", comments)
|
|
313
|
+
}
|
|
314
|
+
if err := second.CancelRun(context.Background(), rid, "repeated cancellation"); err != nil {
|
|
315
|
+
t.Fatalf("repeated cancellation: %v", err)
|
|
316
|
+
}
|
|
317
|
+
if got, err := second.GetRun(context.Background(), rid); err != nil || got.State != run.StateCanceled {
|
|
318
|
+
t.Fatalf("repeated cancellation changed state: run=%+v err=%v", got, err)
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
func TestStartReconcilesCancelingRunWithoutWorkflowInstance(t *testing.T) {
|
|
323
|
+
log := newEventLog()
|
|
324
|
+
sys := newFakeTaskSystem(log)
|
|
325
|
+
fr := newFakeRunner(log)
|
|
326
|
+
deps := goworkflows.Dependencies{
|
|
327
|
+
Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
|
|
328
|
+
Runtime: &run.RuntimePolicy{},
|
|
329
|
+
}
|
|
330
|
+
dbPath := filepath.Join(t.TempDir(), "state.db")
|
|
331
|
+
first, err := goworkflows.New(dbPath, deps)
|
|
332
|
+
if err != nil {
|
|
333
|
+
t.Fatal(err)
|
|
334
|
+
}
|
|
335
|
+
if err := first.Start(context.Background()); err != nil {
|
|
336
|
+
t.Fatal(err)
|
|
337
|
+
}
|
|
338
|
+
rid, err := startRun(first, linearWorkflow(false))
|
|
339
|
+
if err != nil {
|
|
340
|
+
t.Fatal(err)
|
|
341
|
+
}
|
|
342
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
343
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
344
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
345
|
+
})
|
|
346
|
+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
347
|
+
if err := first.Shutdown(shutdownCtx); err != nil {
|
|
348
|
+
t.Fatal(err)
|
|
349
|
+
}
|
|
350
|
+
cancel()
|
|
351
|
+
|
|
352
|
+
db, err := sql.Open("sqlite", dbPath)
|
|
353
|
+
if err != nil {
|
|
354
|
+
t.Fatal(err)
|
|
355
|
+
}
|
|
356
|
+
if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ? WHERE id = ?`,
|
|
357
|
+
string(run.StateCanceling), "operator canceled", string(rid)); err != nil {
|
|
358
|
+
db.Close()
|
|
359
|
+
t.Fatal(err)
|
|
360
|
+
}
|
|
361
|
+
if _, err := db.Exec(`DELETE FROM instances WHERE id = ?`, string(rid)); err != nil {
|
|
362
|
+
db.Close()
|
|
363
|
+
t.Fatal(err)
|
|
364
|
+
}
|
|
365
|
+
if err := db.Close(); err != nil {
|
|
366
|
+
t.Fatal(err)
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
second, err := goworkflows.New(dbPath, deps)
|
|
370
|
+
if err != nil {
|
|
371
|
+
t.Fatal(err)
|
|
372
|
+
}
|
|
373
|
+
if err := second.Start(context.Background()); err != nil {
|
|
374
|
+
t.Fatal(err)
|
|
375
|
+
}
|
|
376
|
+
defer func() { _ = second.Shutdown(context.Background()) }()
|
|
377
|
+
got, err := second.GetRun(context.Background(), rid)
|
|
378
|
+
if err != nil {
|
|
379
|
+
t.Fatal(err)
|
|
380
|
+
}
|
|
381
|
+
if got.State != run.StateCanceled {
|
|
382
|
+
t.Fatalf("startup reconciliation state = %q, want canceled", got.State)
|
|
383
|
+
}
|
|
384
|
+
active, err := second.HasActiveWorkflow(context.Background(), "basicFlow")
|
|
385
|
+
if err != nil {
|
|
386
|
+
t.Fatal(err)
|
|
387
|
+
}
|
|
388
|
+
if active {
|
|
389
|
+
t.Fatal("startup-reconciled run is still counted as active")
|
|
390
|
+
}
|
|
391
|
+
if len(sys.commentBodies("PAY-101")) != 1 {
|
|
392
|
+
t.Fatalf("startup reconciliation comments = %d, want one", len(sys.commentBodies("PAY-101")))
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
func TestStartRetriesCancellationForExistingWorkflowInstance(t *testing.T) {
|
|
397
|
+
log := newEventLog()
|
|
398
|
+
sys := newFakeTaskSystem(log)
|
|
399
|
+
fr := newFakeRunner(log)
|
|
400
|
+
deps := goworkflows.Dependencies{
|
|
401
|
+
Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
|
|
402
|
+
Runtime: &run.RuntimePolicy{},
|
|
403
|
+
}
|
|
404
|
+
dbPath := filepath.Join(t.TempDir(), "state.db")
|
|
405
|
+
first, err := goworkflows.New(dbPath, deps)
|
|
406
|
+
if err != nil {
|
|
407
|
+
t.Fatal(err)
|
|
408
|
+
}
|
|
409
|
+
if err := first.Start(context.Background()); err != nil {
|
|
410
|
+
t.Fatal(err)
|
|
411
|
+
}
|
|
412
|
+
rid, err := startRun(first, linearWorkflow(false))
|
|
413
|
+
if err != nil {
|
|
414
|
+
t.Fatal(err)
|
|
415
|
+
}
|
|
416
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
417
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
418
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
419
|
+
})
|
|
420
|
+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
421
|
+
if err := first.Shutdown(shutdownCtx); err != nil {
|
|
422
|
+
t.Fatal(err)
|
|
423
|
+
}
|
|
424
|
+
cancel()
|
|
425
|
+
|
|
426
|
+
// Leave the engine instance active but persist the cancellation request as
|
|
427
|
+
// if the process died after the projection write and before the cancel RPC.
|
|
428
|
+
db, err := sql.Open("sqlite", dbPath)
|
|
429
|
+
if err != nil {
|
|
430
|
+
t.Fatal(err)
|
|
431
|
+
}
|
|
432
|
+
if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ? WHERE id = ?`,
|
|
433
|
+
string(run.StateCanceling), "startup cancellation", string(rid)); err != nil {
|
|
434
|
+
db.Close()
|
|
435
|
+
t.Fatal(err)
|
|
436
|
+
}
|
|
437
|
+
if err := db.Close(); err != nil {
|
|
438
|
+
t.Fatal(err)
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
second, err := goworkflows.New(dbPath, deps)
|
|
442
|
+
if err != nil {
|
|
443
|
+
t.Fatal(err)
|
|
444
|
+
}
|
|
445
|
+
if err := second.Start(context.Background()); err != nil {
|
|
446
|
+
t.Fatal(err)
|
|
447
|
+
}
|
|
448
|
+
defer func() { _ = second.Shutdown(context.Background()) }()
|
|
449
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
450
|
+
r, _ := second.GetRun(context.Background(), rid)
|
|
451
|
+
return r.State == run.StateCanceled
|
|
452
|
+
})
|
|
453
|
+
if len(sys.commentBodies("PAY-101")) != 1 {
|
|
454
|
+
t.Fatalf("startup cancellation comments = %d, want one", len(sys.commentBodies("PAY-101")))
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
func TestStartDistinguishesFinishedWorkflowFromMissingWorkflow(t *testing.T) {
|
|
459
|
+
log := newEventLog()
|
|
460
|
+
sys := newFakeTaskSystem(log)
|
|
461
|
+
fr := newFakeRunner(log)
|
|
462
|
+
deps := goworkflows.Dependencies{
|
|
463
|
+
Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
|
|
464
|
+
Runtime: &run.RuntimePolicy{},
|
|
465
|
+
}
|
|
466
|
+
dbPath := filepath.Join(t.TempDir(), "state.db")
|
|
467
|
+
first, err := goworkflows.New(dbPath, deps)
|
|
468
|
+
if err != nil {
|
|
469
|
+
t.Fatal(err)
|
|
470
|
+
}
|
|
471
|
+
if err := first.Start(context.Background()); err != nil {
|
|
472
|
+
t.Fatal(err)
|
|
473
|
+
}
|
|
474
|
+
rid, err := startRun(first, linearWorkflow(false))
|
|
475
|
+
if err != nil {
|
|
476
|
+
t.Fatal(err)
|
|
477
|
+
}
|
|
478
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
479
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
480
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
481
|
+
})
|
|
482
|
+
if _, err := first.SubmitReport(context.Background(), reportRequest(rid, "coding", successReport("end"))); err != nil {
|
|
483
|
+
t.Fatal(err)
|
|
484
|
+
}
|
|
485
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
486
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
487
|
+
return r.State == run.StateCompleted
|
|
488
|
+
})
|
|
489
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
490
|
+
db, err := sql.Open("sqlite", dbPath)
|
|
491
|
+
if err != nil {
|
|
492
|
+
return false
|
|
493
|
+
}
|
|
494
|
+
defer db.Close()
|
|
495
|
+
var state int
|
|
496
|
+
if err := db.QueryRow(`SELECT state FROM instances WHERE id = ?`, string(rid)).Scan(&state); err != nil {
|
|
497
|
+
return false
|
|
498
|
+
}
|
|
499
|
+
return state != 0
|
|
500
|
+
})
|
|
501
|
+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
502
|
+
if err := first.Shutdown(shutdownCtx); err != nil {
|
|
503
|
+
t.Fatal(err)
|
|
504
|
+
}
|
|
505
|
+
cancel()
|
|
506
|
+
|
|
507
|
+
// Corrupt only the relay projection into canceling. The finished engine
|
|
508
|
+
// row/history must win; this must not create a cancellation comment.
|
|
509
|
+
db, err := sql.Open("sqlite", dbPath)
|
|
510
|
+
if err != nil {
|
|
511
|
+
t.Fatal(err)
|
|
512
|
+
}
|
|
513
|
+
if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ?, finished_at = NULL WHERE id = ?`,
|
|
514
|
+
string(run.StateCanceling), "late cancellation", string(rid)); err != nil {
|
|
515
|
+
db.Close()
|
|
516
|
+
t.Fatal(err)
|
|
517
|
+
}
|
|
518
|
+
// Make the engine fixture unambiguously finished while retaining its
|
|
519
|
+
// history, so startup must inspect the terminal event rather than treating
|
|
520
|
+
// the row as a missing active execution.
|
|
521
|
+
if _, err := db.Exec(`UPDATE instances SET state = 2, completed_at = CURRENT_TIMESTAMP WHERE id = ?`, string(rid)); err != nil {
|
|
522
|
+
db.Close()
|
|
523
|
+
t.Fatal(err)
|
|
524
|
+
}
|
|
525
|
+
if err := db.Close(); err != nil {
|
|
526
|
+
t.Fatal(err)
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
second, err := goworkflows.New(dbPath, deps)
|
|
530
|
+
if err != nil {
|
|
531
|
+
t.Fatal(err)
|
|
532
|
+
}
|
|
533
|
+
if err := second.Start(context.Background()); err != nil {
|
|
534
|
+
t.Fatal(err)
|
|
535
|
+
}
|
|
536
|
+
defer func() { _ = second.Shutdown(context.Background()) }()
|
|
537
|
+
got, err := second.GetRun(context.Background(), rid)
|
|
538
|
+
if err != nil {
|
|
539
|
+
t.Fatal(err)
|
|
540
|
+
}
|
|
541
|
+
if got.State != run.StateCompleted {
|
|
542
|
+
t.Fatalf("finished workflow was reconciled to %q, want completed", got.State)
|
|
543
|
+
}
|
|
544
|
+
if comments := sys.commentBodies("PAY-101"); len(comments) != 0 {
|
|
545
|
+
t.Fatalf("finished workflow received cancellation comments: %#v", comments)
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
func TestStartReconcilesCanceledWorkflowHistoryAsCanceled(t *testing.T) {
|
|
550
|
+
log := newEventLog()
|
|
551
|
+
sys := newFakeTaskSystem(log)
|
|
552
|
+
fr := newFakeRunner(log)
|
|
553
|
+
deps := goworkflows.Dependencies{
|
|
554
|
+
Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
|
|
555
|
+
Runtime: &run.RuntimePolicy{},
|
|
556
|
+
}
|
|
557
|
+
dbPath := filepath.Join(t.TempDir(), "state.db")
|
|
558
|
+
first, err := goworkflows.New(dbPath, deps)
|
|
559
|
+
if err != nil {
|
|
560
|
+
t.Fatal(err)
|
|
561
|
+
}
|
|
562
|
+
if err := first.Start(context.Background()); err != nil {
|
|
563
|
+
t.Fatal(err)
|
|
564
|
+
}
|
|
565
|
+
rid, err := startRun(first, linearWorkflow(false))
|
|
566
|
+
if err != nil {
|
|
567
|
+
t.Fatal(err)
|
|
568
|
+
}
|
|
569
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
570
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
571
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
572
|
+
})
|
|
573
|
+
if err := first.CancelRun(context.Background(), rid, "history cancellation"); err != nil {
|
|
574
|
+
t.Fatal(err)
|
|
575
|
+
}
|
|
576
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
577
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
578
|
+
return r.State == run.StateCanceled
|
|
579
|
+
})
|
|
580
|
+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
581
|
+
if err := first.Shutdown(shutdownCtx); err != nil {
|
|
582
|
+
t.Fatal(err)
|
|
583
|
+
}
|
|
584
|
+
cancel()
|
|
585
|
+
|
|
586
|
+
db, err := sql.Open("sqlite", dbPath)
|
|
587
|
+
if err != nil {
|
|
588
|
+
t.Fatal(err)
|
|
589
|
+
}
|
|
590
|
+
if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ?, finished_at = NULL WHERE id = ?`,
|
|
591
|
+
string(run.StateCanceling), "history cancellation", string(rid)); err != nil {
|
|
592
|
+
db.Close()
|
|
593
|
+
t.Fatal(err)
|
|
594
|
+
}
|
|
595
|
+
if _, err := db.Exec(`UPDATE instances SET state = 2, completed_at = CURRENT_TIMESTAMP WHERE id = ?`, string(rid)); err != nil {
|
|
596
|
+
db.Close()
|
|
597
|
+
t.Fatal(err)
|
|
598
|
+
}
|
|
599
|
+
if err := db.Close(); err != nil {
|
|
600
|
+
t.Fatal(err)
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
second, err := goworkflows.New(dbPath, deps)
|
|
604
|
+
if err != nil {
|
|
605
|
+
t.Fatal(err)
|
|
606
|
+
}
|
|
607
|
+
if err := second.Start(context.Background()); err != nil {
|
|
608
|
+
t.Fatal(err)
|
|
609
|
+
}
|
|
610
|
+
defer func() { _ = second.Shutdown(context.Background()) }()
|
|
611
|
+
got, err := second.GetRun(context.Background(), rid)
|
|
612
|
+
if err != nil {
|
|
613
|
+
t.Fatal(err)
|
|
614
|
+
}
|
|
615
|
+
if got.State != run.StateCanceled {
|
|
616
|
+
t.Fatalf("canceled workflow history was reconciled to %q, want canceled", got.State)
|
|
617
|
+
}
|
|
618
|
+
if comments := sys.commentBodies("PAY-101"); len(comments) != 1 {
|
|
619
|
+
t.Fatalf("cancellation comments = %d, want one idempotent comment", len(comments))
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
func TestRepeatedCancellationDoesNotAppendCancellationEvents(t *testing.T) {
|
|
624
|
+
log := newEventLog()
|
|
625
|
+
sys := newFakeTaskSystem(log)
|
|
626
|
+
fr := newFakeRunner(log)
|
|
627
|
+
deps := goworkflows.Dependencies{
|
|
628
|
+
Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
|
|
629
|
+
Runtime: &run.RuntimePolicy{},
|
|
630
|
+
}
|
|
631
|
+
dbPath := filepath.Join(t.TempDir(), "state.db")
|
|
632
|
+
first, err := goworkflows.New(dbPath, deps)
|
|
633
|
+
if err != nil {
|
|
634
|
+
t.Fatal(err)
|
|
635
|
+
}
|
|
636
|
+
if err := first.Start(context.Background()); err != nil {
|
|
637
|
+
t.Fatal(err)
|
|
638
|
+
}
|
|
639
|
+
rid, err := startRun(first, linearWorkflow(false))
|
|
640
|
+
if err != nil {
|
|
641
|
+
t.Fatal(err)
|
|
642
|
+
}
|
|
643
|
+
waitFor(t, 10*time.Second, func() bool {
|
|
644
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
645
|
+
return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
|
|
646
|
+
})
|
|
647
|
+
sys.failComments = true
|
|
648
|
+
if err := first.CancelRun(context.Background(), rid, "first reason"); err != nil {
|
|
649
|
+
t.Fatal(err)
|
|
650
|
+
}
|
|
651
|
+
waitFor(t, 15*time.Second, func() bool {
|
|
652
|
+
r, _ := first.GetRun(context.Background(), rid)
|
|
653
|
+
return r.State == run.StateCanceling
|
|
654
|
+
})
|
|
655
|
+
if err := first.CancelRun(context.Background(), rid, "second reason"); err != nil {
|
|
656
|
+
t.Fatal(err)
|
|
657
|
+
}
|
|
658
|
+
if got := countWorkflowEvents(t, dbPath, rid, history.EventType_WorkflowExecutionCanceled); got != 1 {
|
|
659
|
+
t.Fatalf("cancellation events after repeated cancel = %d, want 1", got)
|
|
660
|
+
}
|
|
661
|
+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
662
|
+
if err := first.Shutdown(shutdownCtx); err != nil {
|
|
663
|
+
t.Fatal(err)
|
|
664
|
+
}
|
|
665
|
+
cancel()
|
|
666
|
+
|
|
667
|
+
second, err := goworkflows.New(dbPath, deps)
|
|
668
|
+
if err != nil {
|
|
669
|
+
t.Fatal(err)
|
|
670
|
+
}
|
|
671
|
+
if err := second.Start(context.Background()); err != nil {
|
|
672
|
+
t.Fatal(err)
|
|
673
|
+
}
|
|
674
|
+
defer func() { _ = second.Shutdown(context.Background()) }()
|
|
675
|
+
// Let the already-persisted cancellation event drive cleanup after the
|
|
676
|
+
// restart; startup must not append another event.
|
|
677
|
+
sys.failComments = false
|
|
678
|
+
waitFor(t, 30*time.Second, func() bool {
|
|
679
|
+
r, _ := second.GetRun(context.Background(), rid)
|
|
680
|
+
return r.State == run.StateCanceled
|
|
681
|
+
})
|
|
682
|
+
if got := countWorkflowEvents(t, dbPath, rid, history.EventType_WorkflowExecutionCanceled); got != 1 {
|
|
683
|
+
t.Fatalf("cancellation events after restart = %d, want 1", got)
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
func countWorkflowEvents(t *testing.T, dbPath string, id run.ID, eventType history.EventType) int {
|
|
688
|
+
t.Helper()
|
|
689
|
+
db, err := sql.Open("sqlite", dbPath)
|
|
690
|
+
if err != nil {
|
|
691
|
+
t.Fatal(err)
|
|
692
|
+
}
|
|
693
|
+
defer db.Close()
|
|
694
|
+
var count int
|
|
695
|
+
if err := db.QueryRow(`
|
|
696
|
+
SELECT
|
|
697
|
+
(SELECT COUNT(*) FROM history WHERE instance_id = ? AND event_type = ?)
|
|
698
|
+
+ (SELECT COUNT(*) FROM pending_events WHERE instance_id = ? AND event_type = ?)`,
|
|
699
|
+
string(id), int(eventType), string(id), int(eventType)).Scan(&count); err != nil {
|
|
700
|
+
t.Fatal(err)
|
|
701
|
+
}
|
|
702
|
+
return count
|
|
703
|
+
}
|
|
704
|
+
|
|
223
705
|
func TestExplicitRestartCreatesFreshAttemptFromStart(t *testing.T) {
|
|
224
706
|
log := newEventLog()
|
|
225
707
|
sys := newFakeTaskSystem(log)
|
|
@@ -206,6 +206,44 @@ func TestCompletedRunFinalizesActiveStepWhenFinalUpsertIsMissing(t *testing.T) {
|
|
|
206
206
|
}
|
|
207
207
|
}
|
|
208
208
|
|
|
209
|
+
func TestCancellationFencePreservesReasonAndRejectsLateCompletion(t *testing.T) {
|
|
210
|
+
ctx := context.Background()
|
|
211
|
+
p, _ := openProjection(t)
|
|
212
|
+
start := projectionStart("cancel-fence", "PAY-CANCEL-FENCE")
|
|
213
|
+
if err := p.InsertStart(ctx, start, time.Now().UTC()); err != nil {
|
|
214
|
+
t.Fatal(err)
|
|
215
|
+
}
|
|
216
|
+
first, err := p.BeginCancellation(ctx, start.ID, "first reason")
|
|
217
|
+
if err != nil {
|
|
218
|
+
t.Fatal(err)
|
|
219
|
+
}
|
|
220
|
+
if first.State != run.StateCanceling || first.LastError != "first reason" {
|
|
221
|
+
t.Fatalf("first cancellation = %+v", first)
|
|
222
|
+
}
|
|
223
|
+
second, err := p.BeginCancellation(ctx, start.ID, "replacement reason")
|
|
224
|
+
if err != nil {
|
|
225
|
+
t.Fatal(err)
|
|
226
|
+
}
|
|
227
|
+
if second.State != run.StateCanceling || second.LastError != "first reason" {
|
|
228
|
+
t.Fatalf("repeated cancellation overwrote reason: %+v", second)
|
|
229
|
+
}
|
|
230
|
+
if err := p.UpdateState(ctx, start.ID, run.StateCompleted, "", nil); err != nil {
|
|
231
|
+
t.Fatal(err)
|
|
232
|
+
}
|
|
233
|
+
stillCanceling, err := p.Get(ctx, start.ID)
|
|
234
|
+
if err != nil {
|
|
235
|
+
t.Fatal(err)
|
|
236
|
+
}
|
|
237
|
+
if stillCanceling.State != run.StateCanceling || stillCanceling.LastError != "first reason" {
|
|
238
|
+
t.Fatalf("late completion overwrote cancellation: %+v", stillCanceling)
|
|
239
|
+
}
|
|
240
|
+
finished := time.Now().UTC()
|
|
241
|
+
updated, err := p.UpdateStateIf(ctx, start.ID, run.StateCanceling, run.StateCanceled, "", &finished)
|
|
242
|
+
if err != nil || !updated {
|
|
243
|
+
t.Fatalf("canceling finalization updated=%v err=%v", updated, err)
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
209
247
|
func TestCanceledRunFinalizesCurrentStepTiming(t *testing.T) {
|
|
210
248
|
ctx := context.Background()
|
|
211
249
|
p, _ := openProjection(t)
|
|
@@ -257,24 +257,69 @@ func (p *RunProjection) insertStart(ctx context.Context, s run.Start, now time.T
|
|
|
257
257
|
}
|
|
258
258
|
|
|
259
259
|
func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.State, lastErr string, finished *time.Time) error {
|
|
260
|
+
_, err := p.updateStateCAS(ctx, id, nil, state, lastErr, finished)
|
|
261
|
+
return err
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// beginCancellation atomically claims cancellation of a nonterminal run. A
|
|
265
|
+
// run already in canceling keeps its original reason; completed/canceled runs
|
|
266
|
+
// are never moved backwards. The returned row is read after the conditional
|
|
267
|
+
// update so concurrent cancel/complete callers observe the winner.
|
|
268
|
+
func (p *RunProjection) beginCancellation(ctx context.Context, id run.ID, reason string) (run.Run, error) {
|
|
269
|
+
_, err := p.DB.ExecContext(ctx, `
|
|
270
|
+
UPDATE relay_runs SET state = ?, last_error = ?, updated_at = ?
|
|
271
|
+
WHERE id = ? AND state NOT IN ('completed', 'canceled', 'canceling')`,
|
|
272
|
+
string(run.StateCanceling), reason, time.Now().UTC(), string(id))
|
|
273
|
+
if err != nil {
|
|
274
|
+
return run.Run{}, err
|
|
275
|
+
}
|
|
276
|
+
return p.get(ctx, id)
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
func (p *RunProjection) updateStateCAS(ctx context.Context, id run.ID, expected *run.State, state run.State, lastErr string, finished *time.Time) (bool, error) {
|
|
260
280
|
terminal := state == run.StateCompleted || state == run.StateCanceled
|
|
261
281
|
now := time.Now().UTC()
|
|
262
|
-
|
|
282
|
+
query := `
|
|
263
283
|
UPDATE relay_runs SET state = ?, last_error = ?, updated_at = ?, finished_at = COALESCE(?, finished_at),
|
|
264
284
|
retry_error = CASE WHEN ? THEN NULL ELSE retry_error END,
|
|
265
285
|
retry_attempt = CASE WHEN ? THEN NULL ELSE retry_attempt END,
|
|
266
286
|
next_retry_at = CASE WHEN ? THEN NULL ELSE next_retry_at END
|
|
267
|
-
WHERE id =
|
|
268
|
-
|
|
287
|
+
WHERE id = ?`
|
|
288
|
+
args := []any{
|
|
289
|
+
string(state), lastErr, now, finished, terminal, terminal, terminal,
|
|
290
|
+
string(id),
|
|
291
|
+
}
|
|
292
|
+
if expected == nil {
|
|
293
|
+
// Ordinary workflow projection updates are fenced after cancellation or
|
|
294
|
+
// any terminal state has won. Explicit compare-and-set callers below
|
|
295
|
+
// are allowed to reconcile a known canceling row to its inspected
|
|
296
|
+
// terminal engine result.
|
|
297
|
+
query += ` AND NOT (
|
|
298
|
+
(state = 'canceling' AND ? NOT IN ('canceling', 'canceled'))
|
|
299
|
+
OR (state IN ('completed', 'canceled') AND state <> ?)
|
|
300
|
+
)`
|
|
301
|
+
args = append(args, string(state), string(state))
|
|
302
|
+
} else {
|
|
303
|
+
query += ` AND state = ? AND NOT (state IN ('completed', 'canceled') AND state <> ?)`
|
|
304
|
+
args = append(args, string(*expected), string(state))
|
|
305
|
+
}
|
|
306
|
+
result, err := p.DB.ExecContext(ctx, query, args...)
|
|
269
307
|
if err != nil {
|
|
270
|
-
return err
|
|
308
|
+
return false, err
|
|
309
|
+
}
|
|
310
|
+
updated, err := result.RowsAffected()
|
|
311
|
+
if err != nil {
|
|
312
|
+
return false, err
|
|
313
|
+
}
|
|
314
|
+
if updated != 1 {
|
|
315
|
+
return false, nil
|
|
271
316
|
}
|
|
272
317
|
// Keep the derived timeline useful during retries and cancellation without
|
|
273
318
|
// making it an execution authority. Identify one active row first; a
|
|
274
319
|
// repeated terminal update with no active row is a no-op for the timeline.
|
|
275
320
|
stepStatus := stepStatusForRunState(state)
|
|
276
321
|
if stepStatus == "" {
|
|
277
|
-
return nil
|
|
322
|
+
return true, nil
|
|
278
323
|
}
|
|
279
324
|
var sequence int64
|
|
280
325
|
var startedAt sql.NullTime
|
|
@@ -283,14 +328,14 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
|
|
|
283
328
|
WHERE run_id = ? AND status IN ('running', 'waiting', 'blocked')
|
|
284
329
|
ORDER BY sequence DESC LIMIT 1`, string(id)).Scan(&sequence, &startedAt)
|
|
285
330
|
if errors.Is(lookupErr, sql.ErrNoRows) {
|
|
286
|
-
return nil
|
|
331
|
+
return true, nil
|
|
287
332
|
}
|
|
288
333
|
if lookupErr != nil {
|
|
289
334
|
// relay_run_steps is display/cache data. The authoritative relay_runs
|
|
290
335
|
// state update above has already succeeded and must not be retried just
|
|
291
336
|
// because this optional projection is unavailable.
|
|
292
337
|
slog.Warn("step projection state lookup unavailable", "runID", string(id), "state", state, "error", lookupErr)
|
|
293
|
-
return nil
|
|
338
|
+
return true, nil
|
|
294
339
|
}
|
|
295
340
|
var stepFinished any
|
|
296
341
|
if terminal {
|
|
@@ -300,7 +345,7 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
|
|
|
300
345
|
stepFinished = now
|
|
301
346
|
}
|
|
302
347
|
}
|
|
303
|
-
result, err
|
|
348
|
+
result, err = p.DB.ExecContext(ctx, `
|
|
304
349
|
UPDATE relay_run_steps SET status = ?, message = CASE WHEN ? <> '' THEN ? ELSE message END,
|
|
305
350
|
finished_at = CASE WHEN ? THEN COALESCE(finished_at, ?) ELSE finished_at END
|
|
306
351
|
WHERE run_id = ? AND sequence = ? AND status IN ('running', 'waiting', 'blocked')`,
|
|
@@ -308,11 +353,11 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
|
|
|
308
353
|
string(id), sequence)
|
|
309
354
|
if err != nil {
|
|
310
355
|
slog.Warn("step projection state update unavailable", "runID", string(id), "state", state, "error", err)
|
|
311
|
-
return nil
|
|
356
|
+
return true, nil
|
|
312
357
|
}
|
|
313
|
-
updated, err
|
|
358
|
+
updated, err = result.RowsAffected()
|
|
314
359
|
if err != nil || updated != 1 {
|
|
315
|
-
return nil
|
|
360
|
+
return true, nil
|
|
316
361
|
}
|
|
317
362
|
if terminal && startedAt.Valid {
|
|
318
363
|
endAt := now
|
|
@@ -327,7 +372,7 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
|
|
|
327
372
|
slog.Warn("step projection duration update unavailable", "runID", string(id), "state", state, "error", durationErr)
|
|
328
373
|
}
|
|
329
374
|
}
|
|
330
|
-
return nil
|
|
375
|
+
return true, nil
|
|
331
376
|
}
|
|
332
377
|
|
|
333
378
|
func (p *RunProjection) updateRetry(ctx context.Context, id run.ID, status *run.RetryStatus) error {
|
|
@@ -352,12 +397,20 @@ func (p *RunProjection) updateNode(ctx context.Context, id run.ID, state run.Sta
|
|
|
352
397
|
}
|
|
353
398
|
defer tx.Rollback()
|
|
354
399
|
now := time.Now().UTC()
|
|
355
|
-
|
|
400
|
+
result, err := tx.ExecContext(ctx, `
|
|
356
401
|
UPDATE relay_runs SET state = ?, current_node = ?, current_node_visit_id = ?, updated_at = ?
|
|
357
|
-
WHERE id =
|
|
358
|
-
string(state), node, string(visit), now, string(id))
|
|
402
|
+
WHERE id = ? AND state NOT IN ('canceling', 'completed', 'canceled')`,
|
|
403
|
+
string(state), node, string(visit), now, string(id))
|
|
404
|
+
if err != nil {
|
|
359
405
|
return err
|
|
360
406
|
}
|
|
407
|
+
updated, err := result.RowsAffected()
|
|
408
|
+
if err != nil {
|
|
409
|
+
return err
|
|
410
|
+
}
|
|
411
|
+
if updated != 1 {
|
|
412
|
+
return tx.Commit()
|
|
413
|
+
}
|
|
361
414
|
// A revisit changes only the latest visit ID. Reusable terminal/session
|
|
362
415
|
// identities remain attached to this run/node row.
|
|
363
416
|
if _, err := tx.ExecContext(ctx, `
|
|
@@ -907,6 +960,18 @@ func (p *RunProjection) UpdateState(ctx context.Context, id run.ID, state run.St
|
|
|
907
960
|
return p.updateState(ctx, id, state, lastErr, finished)
|
|
908
961
|
}
|
|
909
962
|
|
|
963
|
+
// BeginCancellation atomically moves a nonterminal run to canceling and
|
|
964
|
+
// returns the persisted row. Existing canceling rows retain their reason.
|
|
965
|
+
func (p *RunProjection) BeginCancellation(ctx context.Context, id run.ID, reason string) (run.Run, error) {
|
|
966
|
+
return p.beginCancellation(ctx, id, reason)
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
// UpdateStateIf applies a lifecycle transition only while the row has the
|
|
970
|
+
// expected state. It returns false when another transition won the race.
|
|
971
|
+
func (p *RunProjection) UpdateStateIf(ctx context.Context, id run.ID, expected, state run.State, lastErr string, finished *time.Time) (bool, error) {
|
|
972
|
+
return p.updateStateCAS(ctx, id, &expected, state, lastErr, finished)
|
|
973
|
+
}
|
|
974
|
+
|
|
910
975
|
// UpdateRetry updates or clears active retry metadata.
|
|
911
976
|
func (p *RunProjection) UpdateRetry(ctx context.Context, id run.ID, status *run.RetryStatus) error {
|
|
912
977
|
return p.updateRetry(ctx, id, status)
|
|
@@ -14,7 +14,7 @@ import (
|
|
|
14
14
|
"github.com/rajpopat27/relay-flow/internal/workflow"
|
|
15
15
|
)
|
|
16
16
|
|
|
17
|
-
const configuredPlugin = "relay-flow-plugin@0.3.
|
|
17
|
+
const configuredPlugin = "relay-flow-plugin@0.3.9-alpha"
|
|
18
18
|
|
|
19
19
|
func TestBuildCommandArgv(t *testing.T) {
|
|
20
20
|
t.Setenv("RELAY_FLOW_HOME", "/var/lib/relay-flow-test")
|