relay-flow 0.3.8-alpha → 0.3.9-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -24,13 +24,13 @@ same `relay-flow-plugin` package with host-specific entrypoints.
24
24
  **OpenCode**
25
25
 
26
26
  ```sh
27
- opencode plugin relay-flow-plugin@0.3.8-alpha
27
+ opencode plugin relay-flow-plugin@0.3.9-alpha
28
28
  ```
29
29
 
30
30
  **Pi**
31
31
 
32
32
  ```sh
33
- pi install npm:relay-flow-plugin@0.3.8-alpha
33
+ pi install npm:relay-flow-plugin@0.3.9-alpha
34
34
  ```
35
35
 
36
36
  Pi loads the package's `pi.ts` extension from its manifest. Do not add
@@ -160,7 +160,7 @@ OpenCode plugin configuration uses both entrypoints. The server entrypoint is li
160
160
  ```json
161
161
  {
162
162
  "$schema": "https://opencode.ai/config.json",
163
- "plugin": ["relay-flow-plugin@0.3.8-alpha"]
163
+ "plugin": ["relay-flow-plugin@0.3.9-alpha"]
164
164
  }
165
165
  ```
166
166
 
@@ -169,7 +169,7 @@ The native HITL approval entrypoint is listed in `.opencode/tui.json`:
169
169
  ```json
170
170
  {
171
171
  "$schema": "https://opencode.ai/tui.json",
172
- "plugin": ["relay-flow-plugin@0.3.8-alpha"]
172
+ "plugin": ["relay-flow-plugin@0.3.9-alpha"]
173
173
  }
174
174
  ```
175
175
 
@@ -185,7 +185,7 @@ Pi plugin: install the same published package manually in Pi's global package
185
185
  settings before starting a Pi harness session:
186
186
 
187
187
  ```sh
188
- pi install npm:relay-flow-plugin@0.3.8-alpha
188
+ pi install npm:relay-flow-plugin@0.3.9-alpha
189
189
  ```
190
190
 
191
191
  Relay-flow does not install or configure the package automatically. Pi resolves
@@ -161,6 +161,17 @@ func (a *Activities) LoadNodeRuntime(ctx context.Context, id run.ID, node string
161
161
  return a.Runs.loadNodeRuntime(ctx, id, node)
162
162
  }
163
163
 
164
+ // LoadCancellationReason reads the operator reason persisted by the
165
+ // cancellation CAS. It is an activity because workflow code cannot access
166
+ // the relay projection directly.
167
+ func (a *Activities) LoadCancellationReason(ctx context.Context, id run.ID) (string, error) {
168
+ r, err := a.Runs.get(ctx, id)
169
+ if err != nil {
170
+ return "", err
171
+ }
172
+ return r.LastError, nil
173
+ }
174
+
164
175
  // EnsureNodeRuntime uses only persisted terminal/session IDs on the normal
165
176
  // path. A live terminal is rebound to the new visit; otherwise EnsureTerminal
166
177
  // creates a replacement and its direct ID is persisted immediately. A stored
@@ -0,0 +1,50 @@
1
+ package goworkflows
2
+
3
+ import (
4
+ "context"
5
+ "database/sql"
6
+ "errors"
7
+ "fmt"
8
+ "testing"
9
+ "time"
10
+
11
+ "github.com/cschleiden/go-workflows/backend"
12
+ )
13
+
14
+ func TestDetachedContextPreservesDeadline(t *testing.T) {
15
+ deadline := time.Now().Add(2 * time.Second)
16
+ parent, cancel := context.WithDeadline(context.Background(), deadline)
17
+ defer cancel()
18
+ detached, cleanup := detachedContext(parent)
19
+ defer cleanup()
20
+ cancel()
21
+ if err := detached.Err(); err != nil {
22
+ t.Fatalf("detached context inherited parent cancellation: %v", err)
23
+ }
24
+ got, ok := detached.Deadline()
25
+ if !ok || got.After(deadline.Add(10*time.Millisecond)) || got.Before(deadline.Add(-10*time.Millisecond)) {
26
+ t.Fatalf("detached deadline = %v, want approximately %v", got, deadline)
27
+ }
28
+ }
29
+
30
+ func TestIsMissingWorkflowInstanceOnlyMatchesConfirmedAbsence(t *testing.T) {
31
+ tests := []struct {
32
+ name string
33
+ err error
34
+ want bool
35
+ }{
36
+ {name: "sql no rows", err: sql.ErrNoRows, want: true},
37
+ {name: "wrapped sql no rows", err: fmt.Errorf("lookup instance: %w", sql.ErrNoRows), want: true},
38
+ {name: "backend not found", err: backend.ErrInstanceNotFound, want: true},
39
+ {name: "wrapped backend not found", err: fmt.Errorf("cancel instance: %w", backend.ErrInstanceNotFound), want: true},
40
+ {name: "database locked", err: errors.New("database is locked"), want: false},
41
+ {name: "context canceled", err: errors.New("context canceled"), want: false},
42
+ }
43
+ for _, tt := range tests {
44
+ t.Run(tt.name, func(t *testing.T) {
45
+ if got := isMissingWorkflowInstance(tt.err); got != tt.want {
46
+ t.Fatalf("isMissingWorkflowInstance(%v) = %v, want %v", tt.err, got, tt.want)
47
+ }
48
+ })
49
+ }
50
+ }
@@ -27,8 +27,10 @@ import (
27
27
  "github.com/rajpopat27/relay-flow/internal/harness"
28
28
  "github.com/rajpopat27/relay-flow/internal/identity"
29
29
  "github.com/rajpopat27/relay-flow/internal/repo"
30
+ "github.com/rajpopat27/relay-flow/internal/retry"
30
31
  "github.com/rajpopat27/relay-flow/internal/run"
31
32
  "github.com/rajpopat27/relay-flow/internal/runner"
33
+ "github.com/rajpopat27/relay-flow/internal/task"
32
34
  "github.com/rajpopat27/relay-flow/internal/workflow"
33
35
  )
34
36
 
@@ -61,6 +63,7 @@ type Engine struct {
61
63
  runtime run.RuntimePolicy
62
64
 
63
65
  mu sync.RWMutex
66
+ cancelMu sync.Mutex // serializes cancellation history check + request
64
67
  snapshots map[run.ID]*workflow.Workflow // in-memory cache; history is authoritative
65
68
 
66
69
  workerCtx context.Context
@@ -177,6 +180,11 @@ func (e *Engine) Start(ctx context.Context) error {
177
180
  if err := e.actWorker.Start(e.workerCtx); err != nil {
178
181
  return fmt.Errorf("start activity worker: %w", err)
179
182
  }
183
+ // A cancellation request can outlive the engine instance that accepted it.
184
+ // Reconcile those projections before normal pollers start; a missing
185
+ // workflow instance is a confirmed terminal execution boundary, while
186
+ // other lookup failures remain retryable and are logged below.
187
+ e.reconcileCancelingRuns(ctx)
180
188
  // Startup retention sweep (pre-poller window): remove old terminal
181
189
  // projection rows and their engine histories; nonterminal runs stay.
182
190
  cutoff := time.Now().Add(-e.retention)
@@ -202,6 +210,7 @@ func (e *Engine) registerActivities() error {
202
210
  a.EnsureEnvironment,
203
211
  a.SetEnvironmentStatus,
204
212
  a.LoadNodeRuntime,
213
+ a.LoadCancellationReason,
205
214
  a.EnsureNodeRuntime,
206
215
  a.CloseTerminals,
207
216
  a.CleanupRun,
@@ -462,26 +471,286 @@ func (e *Engine) workflowOf(ctx context.Context, id run.ID) (*workflow.Workflow,
462
471
  return nil, fmt.Errorf("no workflow snapshot in history for run %s", id)
463
472
  }
464
473
 
465
- // CancelRun cancels the workflow instance; cleanup runs on a disconnected
466
- // workflow context and cannot interrupt an already-running activity.
474
+ // CancelRun requests cancellation of the durable workflow. The projection
475
+ // transition is compare-and-set: a concurrent completion wins if it reaches a
476
+ // terminal state first, while a persisted canceling state is the durable
477
+ // request that startup reconciliation retries.
467
478
  func (e *Engine) CancelRun(ctx context.Context, id run.ID, reason string) error {
468
- if _, err := e.runs.get(ctx, id); err != nil {
479
+ r, err := e.runs.beginCancellation(ctx, id, reason)
480
+ if err != nil {
469
481
  return fmt.Errorf("resolve run %s: %w", id, err)
470
482
  }
471
- if err := e.runs.updateState(ctx, id, run.StateCanceling, reason, nil); err != nil {
483
+ if r.State == run.StateCompleted || r.State == run.StateCanceled {
484
+ return nil
485
+ }
486
+ if r.State != run.StateCanceling {
487
+ return fmt.Errorf("cancel %s: state changed to %s", id, r.State)
488
+ }
489
+ // The reason persisted by the first successful CAS is authoritative for
490
+ // every later cancellation request.
491
+ return e.reconcileCancellation(ctx, r, r.LastError)
492
+ }
493
+
494
+ // isMissingWorkflowInstance identifies only confirmed absence of the active
495
+ // go-workflows execution. Database and context failures remain retryable.
496
+ func isMissingWorkflowInstance(err error) bool {
497
+ return errors.Is(err, sql.ErrNoRows) || errors.Is(err, backend.ErrInstanceNotFound)
498
+ }
499
+
500
+ // detachedContext preserves a caller deadline while detaching cancellation
501
+ // from the caller. Startup cleanup must not be able to outlive its bounded
502
+ // reconciliation window.
503
+ func detachedContext(ctx context.Context) (context.Context, context.CancelFunc) {
504
+ base := context.WithoutCancel(ctx)
505
+ if deadline, ok := ctx.Deadline(); ok {
506
+ return context.WithDeadline(base, deadline)
507
+ }
508
+ return base, func() {}
509
+ }
510
+
511
+ // lookupInstance returns the active execution when present. If only a finished
512
+ // row remains, finished is true; a missing row is the only absence outcome.
513
+ func (e *Engine) lookupInstance(ctx context.Context, id run.ID) (*goworkflow.Instance, bool, error) {
514
+ var execID string
515
+ var state int
516
+ err := e.db.QueryRowContext(ctx, `
517
+ SELECT execution_id, state FROM instances WHERE id = ?
518
+ ORDER BY CASE WHEN state = 0 THEN 0 ELSE 1 END, rowid DESC LIMIT 1`, string(id)).Scan(&execID, &state)
519
+ if err != nil {
520
+ return nil, false, fmt.Errorf("workflow instance %s not found: %w", id, err)
521
+ }
522
+ return &goworkflow.Instance{InstanceID: string(id), ExecutionID: execID}, state != 0, nil
523
+ }
524
+
525
+ // reconcileCancellation retries the durable cancellation request while an
526
+ // execution is active and reconciles finished/missing executions separately.
527
+ func (e *Engine) reconcileCancellation(ctx context.Context, r run.Run, reason string) error {
528
+ inst, finished, err := e.lookupInstance(ctx, r.ID)
529
+ if err != nil {
530
+ if isMissingWorkflowInstance(err) {
531
+ return e.finalizeMissingCancellation(ctx, r, reason)
532
+ }
533
+ return fmt.Errorf("resolve workflow instance: %w", err)
534
+ }
535
+ if finished {
536
+ return e.reconcileFinishedCancellation(ctx, r, inst, reason)
537
+ }
538
+ if e.client == nil {
539
+ return errors.New("workflow engine is not started")
540
+ }
541
+ if err := e.requestCancellation(ctx, inst); err != nil {
542
+ if isMissingWorkflowInstance(err) {
543
+ return e.finalizeMissingCancellation(ctx, r, reason)
544
+ }
472
545
  return err
473
546
  }
474
- inst, err := e.instance(ctx, id)
547
+ return nil
548
+ }
549
+
550
+ // requestCancellation checks durable history and appends at most one
551
+ // cancellation event for an execution. The mutex closes the check/request
552
+ // race between concurrent operator calls in this process; the relay lock
553
+ // prevents another relay-flow server from owning the same database.
554
+ func (e *Engine) requestCancellation(ctx context.Context, inst *goworkflow.Instance) error {
555
+ e.cancelMu.Lock()
556
+ defer e.cancelMu.Unlock()
557
+ var requested int
558
+ if err := e.db.QueryRowContext(ctx, `
559
+ SELECT EXISTS(
560
+ SELECT 1 FROM history
561
+ WHERE instance_id = ? AND execution_id = ? AND event_type = ?
562
+ UNION ALL
563
+ SELECT 1 FROM pending_events
564
+ WHERE instance_id = ? AND execution_id = ? AND event_type = ?
565
+ )`,
566
+ inst.InstanceID, inst.ExecutionID, history.EventType_WorkflowExecutionCanceled,
567
+ inst.InstanceID, inst.ExecutionID, history.EventType_WorkflowExecutionCanceled,
568
+ ).Scan(&requested); err != nil {
569
+ return fmt.Errorf("inspect cancellation history: %w", err)
570
+ }
571
+ if requested != 0 {
572
+ return nil
573
+ }
574
+ return e.client.CancelWorkflowInstance(ctx, inst)
575
+ }
576
+
577
+ // finalizeMissingCancellation performs the same roll-forward cleanup as the
578
+ // workflow's disconnected cancellation path when the workflow instance has
579
+ // genuinely disappeared. It intentionally does not recreate an execution.
580
+ func (e *Engine) finalizeMissingCancellation(ctx context.Context, r run.Run, reason string) error {
581
+ cleanupCtx, cleanupCancel := detachedContext(ctx)
582
+ defer cleanupCancel()
583
+ current, err := e.runs.get(cleanupCtx, r.ID)
584
+ if err != nil {
585
+ return fmt.Errorf("reconcile canceled run %s projection: %w", r.ID, err)
586
+ }
587
+ if current.State == run.StateCompleted || current.State == run.StateCanceled {
588
+ return nil
589
+ }
590
+ if current.State != run.StateCanceling {
591
+ return fmt.Errorf("reconcile canceled run %s: state changed to %s", r.ID, current.State)
592
+ }
593
+ if inst, finished, lookupErr := e.lookupInstance(cleanupCtx, r.ID); lookupErr == nil {
594
+ if finished {
595
+ return e.reconcileFinishedCancellation(cleanupCtx, current, inst, reason)
596
+ }
597
+ return fmt.Errorf("reconcile canceled run %s: workflow instance reappeared", r.ID)
598
+ } else if !isMissingWorkflowInstance(lookupErr) {
599
+ return fmt.Errorf("reconcile canceled run %s instance lookup: %w", r.ID, lookupErr)
600
+ }
601
+ return e.finalizeCancellationEffects(cleanupCtx, current, reason)
602
+ }
603
+
604
+ // finalizeCancellationEffects is shared by missing and already-canceled
605
+ // engine executions. The final projection transition is conditional so a
606
+ // concurrent terminal projection cannot be overwritten.
607
+ func (e *Engine) finalizeCancellationEffects(ctx context.Context, r run.Run, reason string) error {
608
+ if r.State == run.StateCompleted || r.State == run.StateCanceled {
609
+ return nil
610
+ }
611
+ if r.State != run.StateCanceling {
612
+ return fmt.Errorf("finalize canceled run %s: state changed to %s", r.ID, r.State)
613
+ }
614
+ repoInfo, ok := e.activities.Repos.Get(r.Repo)
615
+ if !ok {
616
+ return fmt.Errorf("reconcile canceled run %s: repo %q is no longer registered", r.ID, r.Repo)
617
+ }
618
+ work := run.Work{
619
+ RunID: r.ID,
620
+ LogicalID: r.LogicalID,
621
+ AttemptID: r.AttemptID,
622
+ Repo: r.Repo,
623
+ Workflow: r.Workflow,
624
+ Parent: r.Ticket,
625
+ Runtime: e.runtime,
626
+ }
627
+ if reason == "" {
628
+ reason = "operator requested cancellation"
629
+ }
630
+ if err := e.activities.FinalizeNodeRuntimes(ctx, work, repoInfo.Path, e.runtime); err != nil {
631
+ return fmt.Errorf("finalize canceled run %s terminals: %w", r.ID, err)
632
+ }
633
+ markerID := r.LogicalID
634
+ if markerID == "" {
635
+ markerID = r.ID
636
+ }
637
+ if err := e.activities.Comment(ctx, r.Repo, run.CommentWork{
638
+ RunID: r.ID,
639
+ Item: task.Target{Parent: r.Ticket},
640
+ Body: "Run canceled: " + reason,
641
+ Marker: run.CancellationMarker(markerID),
642
+ }); err != nil {
643
+ return fmt.Errorf("finalize canceled run %s comment: %w", r.ID, err)
644
+ }
645
+ finished := time.Now().UTC()
646
+ updated, err := e.runs.updateStateIf(ctx, r.ID, run.StateCanceling, run.StateCanceled, "", &finished)
475
647
  if err != nil {
476
- return fmt.Errorf("cancel %s: %w", id, err)
648
+ return fmt.Errorf("finalize canceled run %s projection: %w", r.ID, err)
477
649
  }
478
- if err := e.client.CancelWorkflowInstance(ctx, inst); err != nil {
479
- return fmt.Errorf("cancel %s: %w", id, err)
650
+ if !updated {
651
+ latest, getErr := e.runs.get(ctx, r.ID)
652
+ if getErr == nil && (latest.State == run.StateCanceled || latest.State == run.StateCompleted) {
653
+ return nil
654
+ }
655
+ if getErr != nil {
656
+ return fmt.Errorf("finalize canceled run %s projection after race: %w", r.ID, getErr)
657
+ }
658
+ return fmt.Errorf("finalize canceled run %s: state changed to %s", r.ID, latest.State)
480
659
  }
660
+ slog.Info("run canceled", "ticket", r.Ticket.Key, "runID", string(r.ID),
661
+ "repo", r.Repo, "workflow", r.Workflow, "state", run.StateCanceled)
481
662
  return nil
482
663
  }
483
664
 
484
- // instance resolves the current execution for the durable run ID.
665
+ // reconcileFinishedCancellation reads the terminal engine event before
666
+ // deciding whether a canceling projection should become completed or canceled.
667
+ func (e *Engine) reconcileFinishedCancellation(ctx context.Context, r run.Run, inst *goworkflow.Instance, reason string) error {
668
+ if e.backend == nil {
669
+ return errors.New("workflow backend is not started")
670
+ }
671
+ events, err := e.backend.GetWorkflowInstanceHistory(ctx, inst, nil)
672
+ if err != nil {
673
+ return fmt.Errorf("read finished workflow history: %w", err)
674
+ }
675
+ var finishedEvent *history.Event
676
+ canceledBeforeFinish := false
677
+ for _, event := range events {
678
+ switch event.Type {
679
+ case history.EventType_WorkflowExecutionCanceled:
680
+ // TicketWorkflow records this cancellation request first; after
681
+ // cancelCleanup returns, go-workflows may append a normal
682
+ // WorkflowExecutionFinished event. The earlier cancellation event
683
+ // remains authoritative for relay-flow semantics.
684
+ if finishedEvent == nil {
685
+ canceledBeforeFinish = true
686
+ }
687
+ case history.EventType_WorkflowExecutionFinished:
688
+ if finishedEvent == nil {
689
+ finishedEvent = event
690
+ }
691
+ case history.EventType_WorkflowExecutionTerminated:
692
+ return fmt.Errorf("workflow %s terminated without a cancellation or completion result", r.ID)
693
+ }
694
+ }
695
+ if canceledBeforeFinish {
696
+ return e.finalizeCancellationEffects(ctx, r, reason)
697
+ }
698
+ if finishedEvent == nil {
699
+ return fmt.Errorf("finished workflow %s has no terminal history event", r.ID)
700
+ }
701
+ {
702
+ finished := finishedEvent.Timestamp
703
+ updated, err := e.runs.updateStateIf(ctx, r.ID, run.StateCanceling, run.StateCompleted, "", &finished)
704
+ if err != nil {
705
+ return fmt.Errorf("reconcile completed run %s projection: %w", r.ID, err)
706
+ }
707
+ if !updated {
708
+ latest, getErr := e.runs.get(ctx, r.ID)
709
+ if getErr == nil && (latest.State == run.StateCompleted || latest.State == run.StateCanceled) {
710
+ return nil
711
+ }
712
+ if getErr != nil {
713
+ return fmt.Errorf("reconcile completed run %s projection after race: %w", r.ID, getErr)
714
+ }
715
+ return fmt.Errorf("reconcile completed run %s: state changed to %s", r.ID, latest.State)
716
+ }
717
+ return nil
718
+ }
719
+ }
720
+
721
+ // reconcileCancelingRuns retries the durable cancellation request for every
722
+ // stale canceling projection. An active instance is not skipped: its cancel
723
+ // event is re-submitted until the workflow accepts it.
724
+ func (e *Engine) reconcileCancelingRuns(ctx context.Context) {
725
+ active := true
726
+ runs, err := e.runs.list(ctx, run.Filter{Active: &active})
727
+ if err != nil {
728
+ slog.Warn("reconcile canceling runs unavailable", "error", err)
729
+ return
730
+ }
731
+ for _, r := range runs {
732
+ if r.State != run.StateCanceling {
733
+ continue
734
+ }
735
+ if err := e.retryStartupCancellation(ctx, r); err != nil {
736
+ slog.Warn("reconcile canceling run failed", "runID", r.ID, "error", err)
737
+ }
738
+ }
739
+ }
740
+
741
+ // retryStartupCancellation gives a durable canceling projection a bounded
742
+ // startup retry window. Normal polling deliberately does not recreate or
743
+ // advance canceling runs, so an active execution must receive its cancel
744
+ // event here or remain visibly retryable for the next restart.
745
+ func (e *Engine) retryStartupCancellation(ctx context.Context, r run.Run) error {
746
+ retryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
747
+ defer cancel()
748
+ return retry.Do(retryCtx, retry.DefaultBackoffPolicy, func() error {
749
+ return e.reconcileCancellation(retryCtx, r, r.LastError)
750
+ })
751
+ }
752
+
753
+ // instance resolves the current active execution for the durable run ID.
485
754
  func (e *Engine) instance(ctx context.Context, id run.ID) (*goworkflow.Instance, error) {
486
755
  var execID string
487
756
  err := e.db.QueryRowContext(ctx,
@@ -226,7 +226,16 @@ func (s *fakeTaskSystem) Comment(_ context.Context, target task.Target, body, ma
226
226
  if target.Mailbox != nil {
227
227
  key = target.Mailbox.Key
228
228
  }
229
- if s.failComments {
229
+ s.mu.Lock()
230
+ for _, comment := range s.comments {
231
+ if comment.Key == key && comment.Marker == marker {
232
+ s.mu.Unlock()
233
+ return nil
234
+ }
235
+ }
236
+ fail := s.failComments
237
+ s.mu.Unlock()
238
+ if fail {
230
239
  s.log.add("commentFail:" + key)
231
240
  return errTransient
232
241
  }
@@ -75,7 +75,23 @@ func (a *Activities) TicketWorkflow(ctx goworkflow.Context, start run.Start) err
75
75
  WorkflowTaskConfig: start.Workflow.TaskConfig,
76
76
  Runtime: start.Runtime,
77
77
  }
78
- return a.cancelCleanup(ctx, work, start.RepoPath, "canceled")
78
+ // Cancellation events do not carry relay-flow's operator reason. Read
79
+ // the reason persisted by CancelRun through the durable disconnected
80
+ // retry path so cleanup cannot write a permanently wrong comment when
81
+ // the projection is temporarily unavailable.
82
+ dctx := goworkflow.NewDisconnectedContext(ctx)
83
+ reason, reasonErr := retryLoop(dctx, start.ID, a, work, "",
84
+ func(ctx2 goworkflow.Context) goworkflow.Future[string] {
85
+ return goworkflow.ExecuteActivity[string](ctx2, noNativeRetries,
86
+ a.LoadCancellationReason, start.ID)
87
+ })
88
+ if reasonErr != nil {
89
+ return reasonErr
90
+ }
91
+ if reason == "" {
92
+ reason = "canceled"
93
+ }
94
+ return a.cancelCleanup(ctx, work, start.RepoPath, reason)
79
95
  }
80
96
  return err
81
97
  }
@@ -46,6 +46,14 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
46
46
  return p.shared().UpdateState(ctx, id, state, lastErr, finished)
47
47
  }
48
48
 
49
+ func (p *RunProjection) beginCancellation(ctx context.Context, id run.ID, reason string) (run.Run, error) {
50
+ return p.shared().BeginCancellation(ctx, id, reason)
51
+ }
52
+
53
+ func (p *RunProjection) updateStateIf(ctx context.Context, id run.ID, expected, state run.State, lastErr string, finished *time.Time) (bool, error) {
54
+ return p.shared().UpdateStateIf(ctx, id, expected, state, lastErr, finished)
55
+ }
56
+
49
57
  func (p *RunProjection) updateRetry(ctx context.Context, id run.ID, status *run.RetryStatus) error {
50
58
  return p.shared().UpdateRetry(ctx, id, status)
51
59
  }
@@ -9,6 +9,7 @@ import (
9
9
  "testing"
10
10
  "time"
11
11
 
12
+ "github.com/cschleiden/go-workflows/backend/history"
12
13
  "github.com/rajpopat27/relay-flow/internal/execution/goworkflows"
13
14
  "github.com/rajpopat27/relay-flow/internal/identity"
14
15
  recoverpkg "github.com/rajpopat27/relay-flow/internal/recover"
@@ -200,14 +201,19 @@ func TestCancelRun(t *testing.T) {
200
201
  // Exactly one parent cancellation comment with the stable marker.
201
202
  wantMarker := string(rid) + ":cancellation"
202
203
  var cancelComments int
204
+ var cancellationBody string
203
205
  for _, c := range sys.commentBodies("PAY-101") {
204
206
  if c.Marker == wantMarker {
205
207
  cancelComments++
208
+ cancellationBody = c.Body
206
209
  }
207
210
  }
208
211
  if cancelComments != 1 {
209
212
  t.Fatalf("cancellation comments = %d, want 1 with marker %q", cancelComments, wantMarker)
210
213
  }
214
+ if !strings.Contains(cancellationBody, "Run canceled: no longer needed") {
215
+ t.Fatalf("cancellation comment = %q, want persisted operator reason", cancellationBody)
216
+ }
211
217
 
212
218
  // Mailbox statuses/history unchanged: the in-flight coding mailbox was
213
219
  // not completed by cancellation.
@@ -220,6 +226,482 @@ func TestCancelRun(t *testing.T) {
220
226
  }
221
227
  }
222
228
 
229
+ func TestCancelRunFinalizesWhenWorkflowInstanceIsMissing(t *testing.T) {
230
+ log := newEventLog()
231
+ sys := newFakeTaskSystem(log)
232
+ fr := newFakeRunner(log)
233
+ deps := goworkflows.Dependencies{
234
+ Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
235
+ Runtime: &run.RuntimePolicy{},
236
+ }
237
+ dbPath := filepath.Join(t.TempDir(), "state.db")
238
+ first, err := goworkflows.New(dbPath, deps)
239
+ if err != nil {
240
+ t.Fatal(err)
241
+ }
242
+ if err := first.Start(context.Background()); err != nil {
243
+ t.Fatal(err)
244
+ }
245
+ rid, err := startRun(first, linearWorkflow(false))
246
+ if err != nil {
247
+ t.Fatal(err)
248
+ }
249
+ waitFor(t, 10*time.Second, func() bool {
250
+ r, _ := first.GetRun(context.Background(), rid)
251
+ return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
252
+ })
253
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
254
+ if err := first.Shutdown(shutdownCtx); err != nil {
255
+ t.Fatal(err)
256
+ }
257
+ cancel()
258
+
259
+ // Simulate the cancellation crash boundary: the relay projection has a
260
+ // run, but the active go-workflows instance is gone.
261
+ db, err := sql.Open("sqlite", dbPath)
262
+ if err != nil {
263
+ t.Fatal(err)
264
+ }
265
+ if _, err := db.Exec(`DELETE FROM instances WHERE id = ?`, string(rid)); err != nil {
266
+ db.Close()
267
+ t.Fatal(err)
268
+ }
269
+ if err := db.Close(); err != nil {
270
+ t.Fatal(err)
271
+ }
272
+
273
+ second, err := goworkflows.New(dbPath, deps)
274
+ if err != nil {
275
+ t.Fatal(err)
276
+ }
277
+ defer func() { _ = second.Shutdown(context.Background()) }()
278
+ sys.failComments = true
279
+ if err := second.CancelRun(context.Background(), rid, "operator canceled"); err == nil {
280
+ t.Fatal("CancelRun succeeded despite a cancellation-comment failure")
281
+ }
282
+ failed, err := second.GetRun(context.Background(), rid)
283
+ if err != nil {
284
+ t.Fatal(err)
285
+ }
286
+ if failed.State != run.StateCanceling || failed.LastError != "operator canceled" {
287
+ t.Fatalf("failed cancellation projection = %+v, want canceling with original reason", failed)
288
+ }
289
+ sys.failComments = false
290
+ if err := second.CancelRun(context.Background(), rid, "replacement reason"); err != nil {
291
+ t.Fatalf("CancelRun with missing workflow instance: %v", err)
292
+ }
293
+ got, err := second.GetRun(context.Background(), rid)
294
+ if err != nil {
295
+ t.Fatal(err)
296
+ }
297
+ if got.State != run.StateCanceled {
298
+ t.Fatalf("state = %q, want canceled", got.State)
299
+ }
300
+ active, err := second.HasActiveWorkflow(context.Background(), "basicFlow")
301
+ if err != nil {
302
+ t.Fatal(err)
303
+ }
304
+ if active {
305
+ t.Fatal("canceled run is still counted as an active workflow")
306
+ }
307
+ if fr.liveTerminals() != 0 {
308
+ t.Fatalf("missing-instance cancellation left %d live terminals", fr.liveTerminals())
309
+ }
310
+ comments := sys.commentBodies("PAY-101")
311
+ if len(comments) != 1 || comments[0].Marker != string(rid)+":cancellation" {
312
+ t.Fatalf("cancellation comments = %#v, want one stable cancellation marker", comments)
313
+ }
314
+ if err := second.CancelRun(context.Background(), rid, "repeated cancellation"); err != nil {
315
+ t.Fatalf("repeated cancellation: %v", err)
316
+ }
317
+ if got, err := second.GetRun(context.Background(), rid); err != nil || got.State != run.StateCanceled {
318
+ t.Fatalf("repeated cancellation changed state: run=%+v err=%v", got, err)
319
+ }
320
+ }
321
+
322
+ func TestStartReconcilesCancelingRunWithoutWorkflowInstance(t *testing.T) {
323
+ log := newEventLog()
324
+ sys := newFakeTaskSystem(log)
325
+ fr := newFakeRunner(log)
326
+ deps := goworkflows.Dependencies{
327
+ Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
328
+ Runtime: &run.RuntimePolicy{},
329
+ }
330
+ dbPath := filepath.Join(t.TempDir(), "state.db")
331
+ first, err := goworkflows.New(dbPath, deps)
332
+ if err != nil {
333
+ t.Fatal(err)
334
+ }
335
+ if err := first.Start(context.Background()); err != nil {
336
+ t.Fatal(err)
337
+ }
338
+ rid, err := startRun(first, linearWorkflow(false))
339
+ if err != nil {
340
+ t.Fatal(err)
341
+ }
342
+ waitFor(t, 10*time.Second, func() bool {
343
+ r, _ := first.GetRun(context.Background(), rid)
344
+ return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
345
+ })
346
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
347
+ if err := first.Shutdown(shutdownCtx); err != nil {
348
+ t.Fatal(err)
349
+ }
350
+ cancel()
351
+
352
+ db, err := sql.Open("sqlite", dbPath)
353
+ if err != nil {
354
+ t.Fatal(err)
355
+ }
356
+ if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ? WHERE id = ?`,
357
+ string(run.StateCanceling), "operator canceled", string(rid)); err != nil {
358
+ db.Close()
359
+ t.Fatal(err)
360
+ }
361
+ if _, err := db.Exec(`DELETE FROM instances WHERE id = ?`, string(rid)); err != nil {
362
+ db.Close()
363
+ t.Fatal(err)
364
+ }
365
+ if err := db.Close(); err != nil {
366
+ t.Fatal(err)
367
+ }
368
+
369
+ second, err := goworkflows.New(dbPath, deps)
370
+ if err != nil {
371
+ t.Fatal(err)
372
+ }
373
+ if err := second.Start(context.Background()); err != nil {
374
+ t.Fatal(err)
375
+ }
376
+ defer func() { _ = second.Shutdown(context.Background()) }()
377
+ got, err := second.GetRun(context.Background(), rid)
378
+ if err != nil {
379
+ t.Fatal(err)
380
+ }
381
+ if got.State != run.StateCanceled {
382
+ t.Fatalf("startup reconciliation state = %q, want canceled", got.State)
383
+ }
384
+ active, err := second.HasActiveWorkflow(context.Background(), "basicFlow")
385
+ if err != nil {
386
+ t.Fatal(err)
387
+ }
388
+ if active {
389
+ t.Fatal("startup-reconciled run is still counted as active")
390
+ }
391
+ if len(sys.commentBodies("PAY-101")) != 1 {
392
+ t.Fatalf("startup reconciliation comments = %d, want one", len(sys.commentBodies("PAY-101")))
393
+ }
394
+ }
395
+
396
+ func TestStartRetriesCancellationForExistingWorkflowInstance(t *testing.T) {
397
+ log := newEventLog()
398
+ sys := newFakeTaskSystem(log)
399
+ fr := newFakeRunner(log)
400
+ deps := goworkflows.Dependencies{
401
+ Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
402
+ Runtime: &run.RuntimePolicy{},
403
+ }
404
+ dbPath := filepath.Join(t.TempDir(), "state.db")
405
+ first, err := goworkflows.New(dbPath, deps)
406
+ if err != nil {
407
+ t.Fatal(err)
408
+ }
409
+ if err := first.Start(context.Background()); err != nil {
410
+ t.Fatal(err)
411
+ }
412
+ rid, err := startRun(first, linearWorkflow(false))
413
+ if err != nil {
414
+ t.Fatal(err)
415
+ }
416
+ waitFor(t, 10*time.Second, func() bool {
417
+ r, _ := first.GetRun(context.Background(), rid)
418
+ return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
419
+ })
420
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
421
+ if err := first.Shutdown(shutdownCtx); err != nil {
422
+ t.Fatal(err)
423
+ }
424
+ cancel()
425
+
426
+ // Leave the engine instance active but persist the cancellation request as
427
+ // if the process died after the projection write and before the cancel RPC.
428
+ db, err := sql.Open("sqlite", dbPath)
429
+ if err != nil {
430
+ t.Fatal(err)
431
+ }
432
+ if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ? WHERE id = ?`,
433
+ string(run.StateCanceling), "startup cancellation", string(rid)); err != nil {
434
+ db.Close()
435
+ t.Fatal(err)
436
+ }
437
+ if err := db.Close(); err != nil {
438
+ t.Fatal(err)
439
+ }
440
+
441
+ second, err := goworkflows.New(dbPath, deps)
442
+ if err != nil {
443
+ t.Fatal(err)
444
+ }
445
+ if err := second.Start(context.Background()); err != nil {
446
+ t.Fatal(err)
447
+ }
448
+ defer func() { _ = second.Shutdown(context.Background()) }()
449
+ waitFor(t, 30*time.Second, func() bool {
450
+ r, _ := second.GetRun(context.Background(), rid)
451
+ return r.State == run.StateCanceled
452
+ })
453
+ if len(sys.commentBodies("PAY-101")) != 1 {
454
+ t.Fatalf("startup cancellation comments = %d, want one", len(sys.commentBodies("PAY-101")))
455
+ }
456
+ }
457
+
458
+ func TestStartDistinguishesFinishedWorkflowFromMissingWorkflow(t *testing.T) {
459
+ log := newEventLog()
460
+ sys := newFakeTaskSystem(log)
461
+ fr := newFakeRunner(log)
462
+ deps := goworkflows.Dependencies{
463
+ Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
464
+ Runtime: &run.RuntimePolicy{},
465
+ }
466
+ dbPath := filepath.Join(t.TempDir(), "state.db")
467
+ first, err := goworkflows.New(dbPath, deps)
468
+ if err != nil {
469
+ t.Fatal(err)
470
+ }
471
+ if err := first.Start(context.Background()); err != nil {
472
+ t.Fatal(err)
473
+ }
474
+ rid, err := startRun(first, linearWorkflow(false))
475
+ if err != nil {
476
+ t.Fatal(err)
477
+ }
478
+ waitFor(t, 10*time.Second, func() bool {
479
+ r, _ := first.GetRun(context.Background(), rid)
480
+ return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
481
+ })
482
+ if _, err := first.SubmitReport(context.Background(), reportRequest(rid, "coding", successReport("end"))); err != nil {
483
+ t.Fatal(err)
484
+ }
485
+ waitFor(t, 30*time.Second, func() bool {
486
+ r, _ := first.GetRun(context.Background(), rid)
487
+ return r.State == run.StateCompleted
488
+ })
489
+ waitFor(t, 10*time.Second, func() bool {
490
+ db, err := sql.Open("sqlite", dbPath)
491
+ if err != nil {
492
+ return false
493
+ }
494
+ defer db.Close()
495
+ var state int
496
+ if err := db.QueryRow(`SELECT state FROM instances WHERE id = ?`, string(rid)).Scan(&state); err != nil {
497
+ return false
498
+ }
499
+ return state != 0
500
+ })
501
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
502
+ if err := first.Shutdown(shutdownCtx); err != nil {
503
+ t.Fatal(err)
504
+ }
505
+ cancel()
506
+
507
+ // Corrupt only the relay projection into canceling. The finished engine
508
+ // row/history must win; this must not create a cancellation comment.
509
+ db, err := sql.Open("sqlite", dbPath)
510
+ if err != nil {
511
+ t.Fatal(err)
512
+ }
513
+ if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ?, finished_at = NULL WHERE id = ?`,
514
+ string(run.StateCanceling), "late cancellation", string(rid)); err != nil {
515
+ db.Close()
516
+ t.Fatal(err)
517
+ }
518
+ // Make the engine fixture unambiguously finished while retaining its
519
+ // history, so startup must inspect the terminal event rather than treating
520
+ // the row as a missing active execution.
521
+ if _, err := db.Exec(`UPDATE instances SET state = 2, completed_at = CURRENT_TIMESTAMP WHERE id = ?`, string(rid)); err != nil {
522
+ db.Close()
523
+ t.Fatal(err)
524
+ }
525
+ if err := db.Close(); err != nil {
526
+ t.Fatal(err)
527
+ }
528
+
529
+ second, err := goworkflows.New(dbPath, deps)
530
+ if err != nil {
531
+ t.Fatal(err)
532
+ }
533
+ if err := second.Start(context.Background()); err != nil {
534
+ t.Fatal(err)
535
+ }
536
+ defer func() { _ = second.Shutdown(context.Background()) }()
537
+ got, err := second.GetRun(context.Background(), rid)
538
+ if err != nil {
539
+ t.Fatal(err)
540
+ }
541
+ if got.State != run.StateCompleted {
542
+ t.Fatalf("finished workflow was reconciled to %q, want completed", got.State)
543
+ }
544
+ if comments := sys.commentBodies("PAY-101"); len(comments) != 0 {
545
+ t.Fatalf("finished workflow received cancellation comments: %#v", comments)
546
+ }
547
+ }
548
+
549
+ func TestStartReconcilesCanceledWorkflowHistoryAsCanceled(t *testing.T) {
550
+ log := newEventLog()
551
+ sys := newFakeTaskSystem(log)
552
+ fr := newFakeRunner(log)
553
+ deps := goworkflows.Dependencies{
554
+ Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
555
+ Runtime: &run.RuntimePolicy{},
556
+ }
557
+ dbPath := filepath.Join(t.TempDir(), "state.db")
558
+ first, err := goworkflows.New(dbPath, deps)
559
+ if err != nil {
560
+ t.Fatal(err)
561
+ }
562
+ if err := first.Start(context.Background()); err != nil {
563
+ t.Fatal(err)
564
+ }
565
+ rid, err := startRun(first, linearWorkflow(false))
566
+ if err != nil {
567
+ t.Fatal(err)
568
+ }
569
+ waitFor(t, 10*time.Second, func() bool {
570
+ r, _ := first.GetRun(context.Background(), rid)
571
+ return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
572
+ })
573
+ if err := first.CancelRun(context.Background(), rid, "history cancellation"); err != nil {
574
+ t.Fatal(err)
575
+ }
576
+ waitFor(t, 30*time.Second, func() bool {
577
+ r, _ := first.GetRun(context.Background(), rid)
578
+ return r.State == run.StateCanceled
579
+ })
580
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
581
+ if err := first.Shutdown(shutdownCtx); err != nil {
582
+ t.Fatal(err)
583
+ }
584
+ cancel()
585
+
586
+ db, err := sql.Open("sqlite", dbPath)
587
+ if err != nil {
588
+ t.Fatal(err)
589
+ }
590
+ if _, err := db.Exec(`UPDATE relay_runs SET state = ?, last_error = ?, finished_at = NULL WHERE id = ?`,
591
+ string(run.StateCanceling), "history cancellation", string(rid)); err != nil {
592
+ db.Close()
593
+ t.Fatal(err)
594
+ }
595
+ if _, err := db.Exec(`UPDATE instances SET state = 2, completed_at = CURRENT_TIMESTAMP WHERE id = ?`, string(rid)); err != nil {
596
+ db.Close()
597
+ t.Fatal(err)
598
+ }
599
+ if err := db.Close(); err != nil {
600
+ t.Fatal(err)
601
+ }
602
+
603
+ second, err := goworkflows.New(dbPath, deps)
604
+ if err != nil {
605
+ t.Fatal(err)
606
+ }
607
+ if err := second.Start(context.Background()); err != nil {
608
+ t.Fatal(err)
609
+ }
610
+ defer func() { _ = second.Shutdown(context.Background()) }()
611
+ got, err := second.GetRun(context.Background(), rid)
612
+ if err != nil {
613
+ t.Fatal(err)
614
+ }
615
+ if got.State != run.StateCanceled {
616
+ t.Fatalf("canceled workflow history was reconciled to %q, want canceled", got.State)
617
+ }
618
+ if comments := sys.commentBodies("PAY-101"); len(comments) != 1 {
619
+ t.Fatalf("cancellation comments = %d, want one idempotent comment", len(comments))
620
+ }
621
+ }
622
+
623
+ func TestRepeatedCancellationDoesNotAppendCancellationEvents(t *testing.T) {
624
+ log := newEventLog()
625
+ sys := newFakeTaskSystem(log)
626
+ fr := newFakeRunner(log)
627
+ deps := goworkflows.Dependencies{
628
+ Repos: repoRegistryWith("payments", sys), Runner: fr, Harness: newFakeHarness(log),
629
+ Runtime: &run.RuntimePolicy{},
630
+ }
631
+ dbPath := filepath.Join(t.TempDir(), "state.db")
632
+ first, err := goworkflows.New(dbPath, deps)
633
+ if err != nil {
634
+ t.Fatal(err)
635
+ }
636
+ if err := first.Start(context.Background()); err != nil {
637
+ t.Fatal(err)
638
+ }
639
+ rid, err := startRun(first, linearWorkflow(false))
640
+ if err != nil {
641
+ t.Fatal(err)
642
+ }
643
+ waitFor(t, 10*time.Second, func() bool {
644
+ r, _ := first.GetRun(context.Background(), rid)
645
+ return r.CurrentNode == "coding" && r.CurrentNodeVisitID != ""
646
+ })
647
+ sys.failComments = true
648
+ if err := first.CancelRun(context.Background(), rid, "first reason"); err != nil {
649
+ t.Fatal(err)
650
+ }
651
+ waitFor(t, 15*time.Second, func() bool {
652
+ r, _ := first.GetRun(context.Background(), rid)
653
+ return r.State == run.StateCanceling
654
+ })
655
+ if err := first.CancelRun(context.Background(), rid, "second reason"); err != nil {
656
+ t.Fatal(err)
657
+ }
658
+ if got := countWorkflowEvents(t, dbPath, rid, history.EventType_WorkflowExecutionCanceled); got != 1 {
659
+ t.Fatalf("cancellation events after repeated cancel = %d, want 1", got)
660
+ }
661
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
662
+ if err := first.Shutdown(shutdownCtx); err != nil {
663
+ t.Fatal(err)
664
+ }
665
+ cancel()
666
+
667
+ second, err := goworkflows.New(dbPath, deps)
668
+ if err != nil {
669
+ t.Fatal(err)
670
+ }
671
+ if err := second.Start(context.Background()); err != nil {
672
+ t.Fatal(err)
673
+ }
674
+ defer func() { _ = second.Shutdown(context.Background()) }()
675
+ // Let the already-persisted cancellation event drive cleanup after the
676
+ // restart; startup must not append another event.
677
+ sys.failComments = false
678
+ waitFor(t, 30*time.Second, func() bool {
679
+ r, _ := second.GetRun(context.Background(), rid)
680
+ return r.State == run.StateCanceled
681
+ })
682
+ if got := countWorkflowEvents(t, dbPath, rid, history.EventType_WorkflowExecutionCanceled); got != 1 {
683
+ t.Fatalf("cancellation events after restart = %d, want 1", got)
684
+ }
685
+ }
686
+
687
+ func countWorkflowEvents(t *testing.T, dbPath string, id run.ID, eventType history.EventType) int {
688
+ t.Helper()
689
+ db, err := sql.Open("sqlite", dbPath)
690
+ if err != nil {
691
+ t.Fatal(err)
692
+ }
693
+ defer db.Close()
694
+ var count int
695
+ if err := db.QueryRow(`
696
+ SELECT
697
+ (SELECT COUNT(*) FROM history WHERE instance_id = ? AND event_type = ?)
698
+ + (SELECT COUNT(*) FROM pending_events WHERE instance_id = ? AND event_type = ?)`,
699
+ string(id), int(eventType), string(id), int(eventType)).Scan(&count); err != nil {
700
+ t.Fatal(err)
701
+ }
702
+ return count
703
+ }
704
+
223
705
  func TestExplicitRestartCreatesFreshAttemptFromStart(t *testing.T) {
224
706
  log := newEventLog()
225
707
  sys := newFakeTaskSystem(log)
@@ -206,6 +206,44 @@ func TestCompletedRunFinalizesActiveStepWhenFinalUpsertIsMissing(t *testing.T) {
206
206
  }
207
207
  }
208
208
 
209
+ func TestCancellationFencePreservesReasonAndRejectsLateCompletion(t *testing.T) {
210
+ ctx := context.Background()
211
+ p, _ := openProjection(t)
212
+ start := projectionStart("cancel-fence", "PAY-CANCEL-FENCE")
213
+ if err := p.InsertStart(ctx, start, time.Now().UTC()); err != nil {
214
+ t.Fatal(err)
215
+ }
216
+ first, err := p.BeginCancellation(ctx, start.ID, "first reason")
217
+ if err != nil {
218
+ t.Fatal(err)
219
+ }
220
+ if first.State != run.StateCanceling || first.LastError != "first reason" {
221
+ t.Fatalf("first cancellation = %+v", first)
222
+ }
223
+ second, err := p.BeginCancellation(ctx, start.ID, "replacement reason")
224
+ if err != nil {
225
+ t.Fatal(err)
226
+ }
227
+ if second.State != run.StateCanceling || second.LastError != "first reason" {
228
+ t.Fatalf("repeated cancellation overwrote reason: %+v", second)
229
+ }
230
+ if err := p.UpdateState(ctx, start.ID, run.StateCompleted, "", nil); err != nil {
231
+ t.Fatal(err)
232
+ }
233
+ stillCanceling, err := p.Get(ctx, start.ID)
234
+ if err != nil {
235
+ t.Fatal(err)
236
+ }
237
+ if stillCanceling.State != run.StateCanceling || stillCanceling.LastError != "first reason" {
238
+ t.Fatalf("late completion overwrote cancellation: %+v", stillCanceling)
239
+ }
240
+ finished := time.Now().UTC()
241
+ updated, err := p.UpdateStateIf(ctx, start.ID, run.StateCanceling, run.StateCanceled, "", &finished)
242
+ if err != nil || !updated {
243
+ t.Fatalf("canceling finalization updated=%v err=%v", updated, err)
244
+ }
245
+ }
246
+
209
247
  func TestCanceledRunFinalizesCurrentStepTiming(t *testing.T) {
210
248
  ctx := context.Background()
211
249
  p, _ := openProjection(t)
@@ -257,24 +257,69 @@ func (p *RunProjection) insertStart(ctx context.Context, s run.Start, now time.T
257
257
  }
258
258
 
259
259
  func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.State, lastErr string, finished *time.Time) error {
260
+ _, err := p.updateStateCAS(ctx, id, nil, state, lastErr, finished)
261
+ return err
262
+ }
263
+
264
+ // beginCancellation atomically claims cancellation of a nonterminal run. A
265
+ // run already in canceling keeps its original reason; completed/canceled runs
266
+ // are never moved backwards. The returned row is read after the conditional
267
+ // update so concurrent cancel/complete callers observe the winner.
268
+ func (p *RunProjection) beginCancellation(ctx context.Context, id run.ID, reason string) (run.Run, error) {
269
+ _, err := p.DB.ExecContext(ctx, `
270
+ UPDATE relay_runs SET state = ?, last_error = ?, updated_at = ?
271
+ WHERE id = ? AND state NOT IN ('completed', 'canceled', 'canceling')`,
272
+ string(run.StateCanceling), reason, time.Now().UTC(), string(id))
273
+ if err != nil {
274
+ return run.Run{}, err
275
+ }
276
+ return p.get(ctx, id)
277
+ }
278
+
279
+ func (p *RunProjection) updateStateCAS(ctx context.Context, id run.ID, expected *run.State, state run.State, lastErr string, finished *time.Time) (bool, error) {
260
280
  terminal := state == run.StateCompleted || state == run.StateCanceled
261
281
  now := time.Now().UTC()
262
- _, err := p.DB.ExecContext(ctx, `
282
+ query := `
263
283
  UPDATE relay_runs SET state = ?, last_error = ?, updated_at = ?, finished_at = COALESCE(?, finished_at),
264
284
  retry_error = CASE WHEN ? THEN NULL ELSE retry_error END,
265
285
  retry_attempt = CASE WHEN ? THEN NULL ELSE retry_attempt END,
266
286
  next_retry_at = CASE WHEN ? THEN NULL ELSE next_retry_at END
267
- WHERE id = ?`,
268
- string(state), lastErr, now, finished, terminal, terminal, terminal, string(id))
287
+ WHERE id = ?`
288
+ args := []any{
289
+ string(state), lastErr, now, finished, terminal, terminal, terminal,
290
+ string(id),
291
+ }
292
+ if expected == nil {
293
+ // Ordinary workflow projection updates are fenced after cancellation or
294
+ // any terminal state has won. Explicit compare-and-set callers below
295
+ // are allowed to reconcile a known canceling row to its inspected
296
+ // terminal engine result.
297
+ query += ` AND NOT (
298
+ (state = 'canceling' AND ? NOT IN ('canceling', 'canceled'))
299
+ OR (state IN ('completed', 'canceled') AND state <> ?)
300
+ )`
301
+ args = append(args, string(state), string(state))
302
+ } else {
303
+ query += ` AND state = ? AND NOT (state IN ('completed', 'canceled') AND state <> ?)`
304
+ args = append(args, string(*expected), string(state))
305
+ }
306
+ result, err := p.DB.ExecContext(ctx, query, args...)
269
307
  if err != nil {
270
- return err
308
+ return false, err
309
+ }
310
+ updated, err := result.RowsAffected()
311
+ if err != nil {
312
+ return false, err
313
+ }
314
+ if updated != 1 {
315
+ return false, nil
271
316
  }
272
317
  // Keep the derived timeline useful during retries and cancellation without
273
318
  // making it an execution authority. Identify one active row first; a
274
319
  // repeated terminal update with no active row is a no-op for the timeline.
275
320
  stepStatus := stepStatusForRunState(state)
276
321
  if stepStatus == "" {
277
- return nil
322
+ return true, nil
278
323
  }
279
324
  var sequence int64
280
325
  var startedAt sql.NullTime
@@ -283,14 +328,14 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
283
328
  WHERE run_id = ? AND status IN ('running', 'waiting', 'blocked')
284
329
  ORDER BY sequence DESC LIMIT 1`, string(id)).Scan(&sequence, &startedAt)
285
330
  if errors.Is(lookupErr, sql.ErrNoRows) {
286
- return nil
331
+ return true, nil
287
332
  }
288
333
  if lookupErr != nil {
289
334
  // relay_run_steps is display/cache data. The authoritative relay_runs
290
335
  // state update above has already succeeded and must not be retried just
291
336
  // because this optional projection is unavailable.
292
337
  slog.Warn("step projection state lookup unavailable", "runID", string(id), "state", state, "error", lookupErr)
293
- return nil
338
+ return true, nil
294
339
  }
295
340
  var stepFinished any
296
341
  if terminal {
@@ -300,7 +345,7 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
300
345
  stepFinished = now
301
346
  }
302
347
  }
303
- result, err := p.DB.ExecContext(ctx, `
348
+ result, err = p.DB.ExecContext(ctx, `
304
349
  UPDATE relay_run_steps SET status = ?, message = CASE WHEN ? <> '' THEN ? ELSE message END,
305
350
  finished_at = CASE WHEN ? THEN COALESCE(finished_at, ?) ELSE finished_at END
306
351
  WHERE run_id = ? AND sequence = ? AND status IN ('running', 'waiting', 'blocked')`,
@@ -308,11 +353,11 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
308
353
  string(id), sequence)
309
354
  if err != nil {
310
355
  slog.Warn("step projection state update unavailable", "runID", string(id), "state", state, "error", err)
311
- return nil
356
+ return true, nil
312
357
  }
313
- updated, err := result.RowsAffected()
358
+ updated, err = result.RowsAffected()
314
359
  if err != nil || updated != 1 {
315
- return nil
360
+ return true, nil
316
361
  }
317
362
  if terminal && startedAt.Valid {
318
363
  endAt := now
@@ -327,7 +372,7 @@ func (p *RunProjection) updateState(ctx context.Context, id run.ID, state run.St
327
372
  slog.Warn("step projection duration update unavailable", "runID", string(id), "state", state, "error", durationErr)
328
373
  }
329
374
  }
330
- return nil
375
+ return true, nil
331
376
  }
332
377
 
333
378
  func (p *RunProjection) updateRetry(ctx context.Context, id run.ID, status *run.RetryStatus) error {
@@ -352,12 +397,20 @@ func (p *RunProjection) updateNode(ctx context.Context, id run.ID, state run.Sta
352
397
  }
353
398
  defer tx.Rollback()
354
399
  now := time.Now().UTC()
355
- if _, err := tx.ExecContext(ctx, `
400
+ result, err := tx.ExecContext(ctx, `
356
401
  UPDATE relay_runs SET state = ?, current_node = ?, current_node_visit_id = ?, updated_at = ?
357
- WHERE id = ?`,
358
- string(state), node, string(visit), now, string(id)); err != nil {
402
+ WHERE id = ? AND state NOT IN ('canceling', 'completed', 'canceled')`,
403
+ string(state), node, string(visit), now, string(id))
404
+ if err != nil {
359
405
  return err
360
406
  }
407
+ updated, err := result.RowsAffected()
408
+ if err != nil {
409
+ return err
410
+ }
411
+ if updated != 1 {
412
+ return tx.Commit()
413
+ }
361
414
  // A revisit changes only the latest visit ID. Reusable terminal/session
362
415
  // identities remain attached to this run/node row.
363
416
  if _, err := tx.ExecContext(ctx, `
@@ -907,6 +960,18 @@ func (p *RunProjection) UpdateState(ctx context.Context, id run.ID, state run.St
907
960
  return p.updateState(ctx, id, state, lastErr, finished)
908
961
  }
909
962
 
963
+ // BeginCancellation atomically moves a nonterminal run to canceling and
964
+ // returns the persisted row. Existing canceling rows retain their reason.
965
+ func (p *RunProjection) BeginCancellation(ctx context.Context, id run.ID, reason string) (run.Run, error) {
966
+ return p.beginCancellation(ctx, id, reason)
967
+ }
968
+
969
+ // UpdateStateIf applies a lifecycle transition only while the row has the
970
+ // expected state. It returns false when another transition won the race.
971
+ func (p *RunProjection) UpdateStateIf(ctx context.Context, id run.ID, expected, state run.State, lastErr string, finished *time.Time) (bool, error) {
972
+ return p.updateStateCAS(ctx, id, &expected, state, lastErr, finished)
973
+ }
974
+
910
975
  // UpdateRetry updates or clears active retry metadata.
911
976
  func (p *RunProjection) UpdateRetry(ctx context.Context, id run.ID, status *run.RetryStatus) error {
912
977
  return p.updateRetry(ctx, id, status)
@@ -14,7 +14,7 @@ import (
14
14
  "github.com/rajpopat27/relay-flow/internal/workflow"
15
15
  )
16
16
 
17
- const configuredPlugin = "relay-flow-plugin@0.3.8-alpha"
17
+ const configuredPlugin = "relay-flow-plugin@0.3.9-alpha"
18
18
 
19
19
  func TestBuildCommandArgv(t *testing.T) {
20
20
  t.Setenv("RELAY_FLOW_HOME", "/var/lib/relay-flow-test")
@@ -10,7 +10,7 @@ import (
10
10
  "github.com/rajpopat27/relay-flow/internal/config"
11
11
  )
12
12
 
13
- const relayFlowPlugin = "relay-flow-plugin@0.3.8-alpha"
13
+ const relayFlowPlugin = "relay-flow-plugin@0.3.9-alpha"
14
14
 
15
15
  type jsoncToken struct {
16
16
  kind byte
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "relay-flow",
3
- "version": "0.3.8-alpha",
3
+ "version": "0.3.9-alpha",
4
4
  "description": "Graph-based agent workflow engine — tracker-agnostic, pluggable runners",
5
5
  "bin": {
6
6
  "relay-flow": "./bin/relay-flow.js",