relay-flow 0.3.8-alpha → 0.3.10-alpha

package/internal/execution/goworkflows/engine.go

@@ -27,8 +27,10 @@ import (
 	"github.com/rajpopat27/relay-flow/internal/harness"
 	"github.com/rajpopat27/relay-flow/internal/identity"
 	"github.com/rajpopat27/relay-flow/internal/repo"
+	"github.com/rajpopat27/relay-flow/internal/retry"
 	"github.com/rajpopat27/relay-flow/internal/run"
 	"github.com/rajpopat27/relay-flow/internal/runner"
+	"github.com/rajpopat27/relay-flow/internal/task"
 	"github.com/rajpopat27/relay-flow/internal/workflow"
 )
 
@@ -61,6 +63,7 @@ type Engine struct {
 	runtime    run.RuntimePolicy
 
 	mu        sync.RWMutex
+	cancelMu  sync.Mutex                    // serializes cancellation history check + request
 	snapshots map[run.ID]*workflow.Workflow // in-memory cache; history is authoritative
 
 	workerCtx    context.Context
@@ -177,6 +180,11 @@ func (e *Engine) Start(ctx context.Context) error {
 	if err := e.actWorker.Start(e.workerCtx); err != nil {
 		return fmt.Errorf("start activity worker: %w", err)
 	}
+	// A cancellation request can outlive the engine instance that accepted it.
+	// Reconcile those projections before normal pollers start; a missing
+	// workflow instance is a confirmed terminal execution boundary, while
+	// other lookup failures remain retryable and are logged below.
+	e.reconcileCancelingRuns(ctx)
 	// Startup retention sweep (pre-poller window): remove old terminal
 	// projection rows and their engine histories; nonterminal runs stay.
 	cutoff := time.Now().Add(-e.retention)
@@ -202,6 +210,7 @@ func (e *Engine) registerActivities() error {
 		a.EnsureEnvironment,
 		a.SetEnvironmentStatus,
 		a.LoadNodeRuntime,
+		a.LoadCancellationReason,
 		a.EnsureNodeRuntime,
 		a.CloseTerminals,
 		a.CleanupRun,
@@ -462,26 +471,286 @@ func (e *Engine) workflowOf(ctx context.Context, id run.ID) (*workflow.Workflow,
 	return nil, fmt.Errorf("no workflow snapshot in history for run %s", id)
 }
 
-// CancelRun cancels the workflow instance; cleanup runs on a disconnected
-// workflow context and cannot interrupt an already-running activity.
+// CancelRun requests cancellation of the durable workflow. The projection
+// transition is compare-and-set: a concurrent completion wins if it reaches a
+// terminal state first, while a persisted canceling state is the durable
+// request that startup reconciliation retries.
 func (e *Engine) CancelRun(ctx context.Context, id run.ID, reason string) error {
-	if _, err := e.runs.get(ctx, id); err != nil {
+	r, err := e.runs.beginCancellation(ctx, id, reason)
+	if err != nil {
 		return fmt.Errorf("resolve run %s: %w", id, err)
 	}
-	if err := e.runs.updateState(ctx, id, run.StateCanceling, reason, nil); err != nil {
+	if r.State == run.StateCompleted || r.State == run.StateCanceled {
+		return nil
+	}
+	if r.State != run.StateCanceling {
+		return fmt.Errorf("cancel %s: state changed to %s", id, r.State)
+	}
+	// The reason persisted by the first successful CAS is authoritative for
+	// every later cancellation request.
+	return e.reconcileCancellation(ctx, r, r.LastError)
+}
+
+// isMissingWorkflowInstance identifies only confirmed absence of the active
+// go-workflows execution. Database and context failures remain retryable.
+func isMissingWorkflowInstance(err error) bool {
+	return errors.Is(err, sql.ErrNoRows) || errors.Is(err, backend.ErrInstanceNotFound)
+}
+
+// detachedContext preserves a caller deadline while detaching cancellation
+// from the caller. Startup cleanup must not be able to outlive its bounded
+// reconciliation window.
+func detachedContext(ctx context.Context) (context.Context, context.CancelFunc) {
+	base := context.WithoutCancel(ctx)
+	if deadline, ok := ctx.Deadline(); ok {
+		return context.WithDeadline(base, deadline)
+	}
+	return base, func() {}
+}
+
+// lookupInstance returns the active execution when present. If only a finished
+// row remains, finished is true; a missing row is the only absence outcome.
+func (e *Engine) lookupInstance(ctx context.Context, id run.ID) (*goworkflow.Instance, bool, error) {
+	var execID string
+	var state int
+	err := e.db.QueryRowContext(ctx, `
+		SELECT execution_id, state FROM instances WHERE id = ?
+		ORDER BY CASE WHEN state = 0 THEN 0 ELSE 1 END, rowid DESC LIMIT 1`, string(id)).Scan(&execID, &state)
+	if err != nil {
+		return nil, false, fmt.Errorf("workflow instance %s not found: %w", id, err)
+	}
+	return &goworkflow.Instance{InstanceID: string(id), ExecutionID: execID}, state != 0, nil
+}
+
+// reconcileCancellation retries the durable cancellation request while an
+// execution is active and reconciles finished/missing executions separately.
+func (e *Engine) reconcileCancellation(ctx context.Context, r run.Run, reason string) error {
+	inst, finished, err := e.lookupInstance(ctx, r.ID)
+	if err != nil {
+		if isMissingWorkflowInstance(err) {
+			return e.finalizeMissingCancellation(ctx, r, reason)
+		}
+		return fmt.Errorf("resolve workflow instance: %w", err)
+	}
+	if finished {
+		return e.reconcileFinishedCancellation(ctx, r, inst, reason)
+	}
+	if e.client == nil {
+		return errors.New("workflow engine is not started")
+	}
+	if err := e.requestCancellation(ctx, inst); err != nil {
+		if isMissingWorkflowInstance(err) {
+			return e.finalizeMissingCancellation(ctx, r, reason)
+		}
 		return err
 	}
-	inst, err := e.instance(ctx, id)
+	return nil
+}
+
+// requestCancellation checks durable history and appends at most one
+// cancellation event for an execution. The mutex closes the check/request
+// race between concurrent operator calls in this process; the relay lock
+// prevents another relay-flow server from owning the same database.
+func (e *Engine) requestCancellation(ctx context.Context, inst *goworkflow.Instance) error {
+	e.cancelMu.Lock()
+	defer e.cancelMu.Unlock()
+	var requested int
+	if err := e.db.QueryRowContext(ctx, `
+		SELECT EXISTS(
+			SELECT 1 FROM history
+			WHERE instance_id = ? AND execution_id = ? AND event_type = ?
+			UNION ALL
+			SELECT 1 FROM pending_events
+			WHERE instance_id = ? AND execution_id = ? AND event_type = ?
+		)`,
+		inst.InstanceID, inst.ExecutionID, history.EventType_WorkflowExecutionCanceled,
+		inst.InstanceID, inst.ExecutionID, history.EventType_WorkflowExecutionCanceled,
+	).Scan(&requested); err != nil {
+		return fmt.Errorf("inspect cancellation history: %w", err)
+	}
+	if requested != 0 {
+		return nil
+	}
+	return e.client.CancelWorkflowInstance(ctx, inst)
+}
+
+// finalizeMissingCancellation performs the same roll-forward cleanup as the
+// workflow's disconnected cancellation path when the workflow instance has
+// genuinely disappeared. It intentionally does not recreate an execution.
+func (e *Engine) finalizeMissingCancellation(ctx context.Context, r run.Run, reason string) error {
+	cleanupCtx, cleanupCancel := detachedContext(ctx)
+	defer cleanupCancel()
+	current, err := e.runs.get(cleanupCtx, r.ID)
+	if err != nil {
+		return fmt.Errorf("reconcile canceled run %s projection: %w", r.ID, err)
+	}
+	if current.State == run.StateCompleted || current.State == run.StateCanceled {
+		return nil
+	}
+	if current.State != run.StateCanceling {
+		return fmt.Errorf("reconcile canceled run %s: state changed to %s", r.ID, current.State)
+	}
+	if inst, finished, lookupErr := e.lookupInstance(cleanupCtx, r.ID); lookupErr == nil {
+		if finished {
+			return e.reconcileFinishedCancellation(cleanupCtx, current, inst, reason)
+		}
+		return fmt.Errorf("reconcile canceled run %s: workflow instance reappeared", r.ID)
+	} else if !isMissingWorkflowInstance(lookupErr) {
+		return fmt.Errorf("reconcile canceled run %s instance lookup: %w", r.ID, lookupErr)
+	}
+	return e.finalizeCancellationEffects(cleanupCtx, current, reason)
+}
+
+// finalizeCancellationEffects is shared by missing and already-canceled
+// engine executions. The final projection transition is conditional so a
+// concurrent terminal projection cannot be overwritten.
+func (e *Engine) finalizeCancellationEffects(ctx context.Context, r run.Run, reason string) error {
+	if r.State == run.StateCompleted || r.State == run.StateCanceled {
+		return nil
+	}
+	if r.State != run.StateCanceling {
+		return fmt.Errorf("finalize canceled run %s: state changed to %s", r.ID, r.State)
+	}
+	repoInfo, ok := e.activities.Repos.Get(r.Repo)
+	if !ok {
+		return fmt.Errorf("reconcile canceled run %s: repo %q is no longer registered", r.ID, r.Repo)
+	}
+	work := run.Work{
+		RunID:     r.ID,
+		LogicalID: r.LogicalID,
+		AttemptID: r.AttemptID,
+		Repo:      r.Repo,
+		Workflow:  r.Workflow,
+		Parent:    r.Ticket,
+		Runtime:   e.runtime,
+	}
+	if reason == "" {
+		reason = "operator requested cancellation"
+	}
+	if err := e.activities.FinalizeNodeRuntimes(ctx, work, repoInfo.Path, e.runtime); err != nil {
+		return fmt.Errorf("finalize canceled run %s terminals: %w", r.ID, err)
+	}
+	markerID := r.LogicalID
+	if markerID == "" {
+		markerID = r.ID
+	}
+	if err := e.activities.Comment(ctx, r.Repo, run.CommentWork{
+		RunID:  r.ID,
+		Item:   task.Target{Parent: r.Ticket},
+		Body:   "Run canceled: " + reason,
+		Marker: run.CancellationMarker(markerID),
+	}); err != nil {
+		return fmt.Errorf("finalize canceled run %s comment: %w", r.ID, err)
+	}
+	finished := time.Now().UTC()
+	updated, err := e.runs.updateStateIf(ctx, r.ID, run.StateCanceling, run.StateCanceled, "", &finished)
 	if err != nil {
-		return fmt.Errorf("cancel %s: %w", id, err)
+		return fmt.Errorf("finalize canceled run %s projection: %w", r.ID, err)
 	}
-	if err := e.client.CancelWorkflowInstance(ctx, inst); err != nil {
-		return fmt.Errorf("cancel %s: %w", id, err)
+	if !updated {
+		latest, getErr := e.runs.get(ctx, r.ID)
+		if getErr == nil && (latest.State == run.StateCanceled || latest.State == run.StateCompleted) {
+			return nil
+		}
+		if getErr != nil {
+			return fmt.Errorf("finalize canceled run %s projection after race: %w", r.ID, getErr)
+		}
+		return fmt.Errorf("finalize canceled run %s: state changed to %s", r.ID, latest.State)
 	}
+	slog.Info("run canceled", "ticket", r.Ticket.Key, "runID", string(r.ID),
+		"repo", r.Repo, "workflow", r.Workflow, "state", run.StateCanceled)
 	return nil
 }
 
-// instance resolves the current execution for the durable run ID.
+// reconcileFinishedCancellation reads the terminal engine event before
+// deciding whether a canceling projection should become completed or canceled.
+func (e *Engine) reconcileFinishedCancellation(ctx context.Context, r run.Run, inst *goworkflow.Instance, reason string) error {
+	if e.backend == nil {
+		return errors.New("workflow backend is not started")
+	}
+	events, err := e.backend.GetWorkflowInstanceHistory(ctx, inst, nil)
+	if err != nil {
+		return fmt.Errorf("read finished workflow history: %w", err)
+	}
+	var finishedEvent *history.Event
+	canceledBeforeFinish := false
+	for _, event := range events {
+		switch event.Type {
+		case history.EventType_WorkflowExecutionCanceled:
+			// TicketWorkflow records this cancellation request first; after
+			// cancelCleanup returns, go-workflows may append a normal
+			// WorkflowExecutionFinished event. The earlier cancellation event
+			// remains authoritative for relay-flow semantics.
+			if finishedEvent == nil {
+				canceledBeforeFinish = true
+			}
+		case history.EventType_WorkflowExecutionFinished:
+			if finishedEvent == nil {
+				finishedEvent = event
+			}
+		case history.EventType_WorkflowExecutionTerminated:
+			return fmt.Errorf("workflow %s terminated without a cancellation or completion result", r.ID)
+		}
+	}
+	if canceledBeforeFinish {
+		return e.finalizeCancellationEffects(ctx, r, reason)
+	}
+	if finishedEvent == nil {
+		return fmt.Errorf("finished workflow %s has no terminal history event", r.ID)
+	}
+	{
+		finished := finishedEvent.Timestamp
+		updated, err := e.runs.updateStateIf(ctx, r.ID, run.StateCanceling, run.StateCompleted, "", &finished)
+		if err != nil {
+			return fmt.Errorf("reconcile completed run %s projection: %w", r.ID, err)
+		}
+		if !updated {
+			latest, getErr := e.runs.get(ctx, r.ID)
+			if getErr == nil && (latest.State == run.StateCompleted || latest.State == run.StateCanceled) {
+				return nil
+			}
+			if getErr != nil {
+				return fmt.Errorf("reconcile completed run %s projection after race: %w", r.ID, getErr)
+			}
+			return fmt.Errorf("reconcile completed run %s: state changed to %s", r.ID, latest.State)
+		}
+		return nil
+	}
+}
+
+// reconcileCancelingRuns retries the durable cancellation request for every
+// stale canceling projection. An active instance is not skipped: its cancel
+// event is re-submitted until the workflow accepts it.
+func (e *Engine) reconcileCancelingRuns(ctx context.Context) {
+	active := true
+	runs, err := e.runs.list(ctx, run.Filter{Active: &active})
+	if err != nil {
+		slog.Warn("reconcile canceling runs unavailable", "error", err)
+		return
+	}
+	for _, r := range runs {
+		if r.State != run.StateCanceling {
+			continue
+		}
+		if err := e.retryStartupCancellation(ctx, r); err != nil {
+			slog.Warn("reconcile canceling run failed", "runID", r.ID, "error", err)
+		}
+	}
+}
+
+// retryStartupCancellation gives a durable canceling projection a bounded
+// startup retry window. Normal polling deliberately does not recreate or
+// advance canceling runs, so an active execution must receive its cancel
+// event here or remain visibly retryable for the next restart.
+func (e *Engine) retryStartupCancellation(ctx context.Context, r run.Run) error {
+	retryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+	defer cancel()
+	return retry.Do(retryCtx, retry.DefaultBackoffPolicy, func() error {
+		return e.reconcileCancellation(retryCtx, r, r.LastError)
+	})
+}
+
+// instance resolves the current active execution for the durable run ID.
 func (e *Engine) instance(ctx context.Context, id run.ID) (*goworkflow.Instance, error) {
 	var execID string
 	err := e.db.QueryRowContext(ctx,

package/internal/execution/goworkflows/fakes_test.go

@@ -73,6 +73,7 @@ type fakeTaskSystem struct {
 	labels        map[string][]string // mailbox key -> labels
 	specs         []task.MailboxSpec
 	comments      []recordedComment
+	hasCommentN   int
 	resets        []string
 	renderText    func(task.TextKind, task.TextData) (string, error)
 
@@ -115,6 +116,18 @@ func (s *fakeTaskSystem) CompileFilter(config.RawValues) (func(task.Ticket) bool
 	return func(task.Ticket) bool { return true }, nil
 }
 
+func (s *fakeTaskSystem) CompileOwnershipFilter(config.RawValues) (func(task.Ticket) bool, error) {
+	return func(task.Ticket) bool { return true }, nil
+}
+
+func (s *fakeTaskSystem) ValidateOwnership(context.Context, task.TicketRef, string, config.RawValues) error {
+	return nil
+}
+
+func (s *fakeTaskSystem) ClaimIfOwned(ctx context.Context, ref task.TicketRef, workflowName string, _ config.RawValues) error {
+	return s.Claim(ctx, ref, workflowName)
+}
+
 func (s *fakeTaskSystem) Claim(_ context.Context, ref task.TicketRef, wf string) error {
 	s.log.add("claim:" + ref.Key + ":" + wf)
 	return nil
@@ -208,6 +221,7 @@ func (s *fakeTaskSystem) CompleteMailbox(_ context.Context, mb task.Mailbox) err
 
 func (s *fakeTaskSystem) HasComment(_ context.Context, target task.Target, marker string) (bool, error) {
 	s.mu.Lock()
+	s.hasCommentN++
 	defer s.mu.Unlock()
 	for _, c := range s.comments {
 		if c.Marker == marker {
@@ -226,7 +240,16 @@ func (s *fakeTaskSystem) Comment(_ context.Context, target task.Target, body, ma
 	if target.Mailbox != nil {
 		key = target.Mailbox.Key
 	}
-	if s.failComments {
+	s.mu.Lock()
+	for _, comment := range s.comments {
+		if comment.Key == key && comment.Marker == marker {
+			s.mu.Unlock()
+			return nil
+		}
+	}
+	fail := s.failComments
+	s.mu.Unlock()
+	if fail {
 		s.log.add("commentFail:" + key)
 		return errTransient
 	}
@@ -258,6 +281,12 @@ func (s *fakeTaskSystem) ResetForRecovery(_ context.Context, parent task.TicketR
 	return nil
 }
 
+func (s *fakeTaskSystem) hasCommentCount() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.hasCommentN
+}
+
 func (s *fakeTaskSystem) commentBodies(key string) []recordedComment {
 	s.mu.Lock()
 	defer s.mu.Unlock()

package/internal/execution/goworkflows/interpreter.go

@@ -75,7 +75,23 @@ func (a *Activities) TicketWorkflow(ctx goworkflow.Context, start run.Start) err
 			WorkflowTaskConfig: start.Workflow.TaskConfig,
 			Runtime:            start.Runtime,
 		}
-		return a.cancelCleanup(ctx, work, start.RepoPath, "canceled")
+		// Cancellation events do not carry relay-flow's operator reason. Read
+		// the reason persisted by CancelRun through the durable disconnected
+		// retry path so cleanup cannot write a permanently wrong comment when
+		// the projection is temporarily unavailable.
+		dctx := goworkflow.NewDisconnectedContext(ctx)
+		reason, reasonErr := retryLoop(dctx, start.ID, a, work, "",
+			func(ctx2 goworkflow.Context) goworkflow.Future[string] {
+				return goworkflow.ExecuteActivity[string](ctx2, noNativeRetries,
+					a.LoadCancellationReason, start.ID)
+			})
+		if reasonErr != nil {
+			return reasonErr
+		}
+		if reason == "" {
+			reason = "canceled"
+		}
+		return a.cancelCleanup(ctx, work, start.RepoPath, reason)
 	}
 	return err
 }
@@ -276,6 +292,7 @@ func (a *Activities) runGraph(ctx goworkflow.Context, start run.Start) error {
 			Ticket:      start.Ticket.Key,
 			Node:        current,
 			NodeType:    node.Type,
+			AutoReject:  node.AutoReject,
 			Agent:       node.Agent,
 			Title:       title,
 			NudgePrompt: node.NudgePrompt,

package/internal/execution/goworkflows/ownership_test.go

@@ -0,0 +1,259 @@
+package goworkflows_test
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/rajpopat27/relay-flow/internal/config"
+	"github.com/rajpopat27/relay-flow/internal/execution/goworkflows"
+	"github.com/rajpopat27/relay-flow/internal/identity"
+	recoverpkg "github.com/rajpopat27/relay-flow/internal/recover"
+	"github.com/rajpopat27/relay-flow/internal/router"
+	"github.com/rajpopat27/relay-flow/internal/run"
+	"github.com/rajpopat27/relay-flow/internal/task"
+	"github.com/rajpopat27/relay-flow/internal/workflow"
+)
+
+// ownershipCompositionState models the provider-owned labels and current
+// assignee shared by two independent relay-flow databases. The adapter fake
+// persists a durable wf-owner marker when the first server claims the ticket.
+type ownershipCompositionState struct {
+	mu       sync.Mutex
+	assignee string
+	labels   []string
+}
+
+type ownershipCompositionTaskSystem struct {
+	*fakeTaskSystem
+	owner string
+	state *ownershipCompositionState
+}
+
+func newOwnershipCompositionTaskSystem(owner string, state *ownershipCompositionState, log *eventLog) *ownershipCompositionTaskSystem {
+	return &ownershipCompositionTaskSystem{
+		fakeTaskSystem: newFakeTaskSystem(log), owner: owner, state: state,
+	}
+}
+
+func (s *ownershipCompositionTaskSystem) Poll(context.Context) ([]task.Ticket, error) {
+	s.state.mu.Lock()
+	defer s.state.mu.Unlock()
+	claims := make([]string, 0, 1)
+	for _, label := range s.state.labels {
+		if len(label) > len("wf:") && label[:len("wf:")] == "wf:" {
+			claims = append(claims, label)
+		}
+	}
+	return []task.Ticket{{
+		ID: "1", Key: "PAY-101", Title: "ownership composition",
+		WorkflowClaims: claims,
+		Fields: map[string]any{
+			"assignee": s.state.assignee,
+			"labels":   append([]string(nil), s.state.labels...),
+		},
+	}}, nil
+}
+
+func (s *ownershipCompositionTaskSystem) CompileFilter(config.RawValues) (func(task.Ticket) bool, error) {
+	return func(ticket task.Ticket) bool {
+		assignee, _ := ticket.Fields["assignee"].(string)
+		return assignee == s.owner
+	}, nil
+}
+
+func (s *ownershipCompositionTaskSystem) CompileOwnershipFilter(config.RawValues) (func(task.Ticket) bool, error) {
+	return func(ticket task.Ticket) bool {
+		if len(ticket.WorkflowClaims) != 1 {
+			return false
+		}
+		assignee, _ := ticket.Fields["assignee"].(string)
+		if assignee != s.owner {
+			return false
+		}
+		want := "wf-owner:" + ticket.WorkflowClaims[0][len("wf:"):] + ":" + s.owner
+		labels, _ := ticket.Fields["labels"].([]string)
+		for _, label := range labels {
+			if label == want {
+				return true
+			}
+		}
+		return false
+	}, nil
+}
+
+func (s *ownershipCompositionTaskSystem) ValidateOwnership(context.Context, task.TicketRef, string, config.RawValues) error {
+	s.state.mu.Lock()
+	defer s.state.mu.Unlock()
+	if s.hasOwnerMarkerLocked("ownershipFlow") {
+		return nil
+	}
+	return &task.OwnershipMismatchError{Ticket: "PAY-101", Workflow: "ownershipFlow"}
+}
+
+func (s *ownershipCompositionTaskSystem) BackfillClaimOwner(context.Context, task.TicketRef, string, config.RawValues) error {
+	s.state.mu.Lock()
+	defer s.state.mu.Unlock()
+	if !containsLabel(s.state.labels, "wf:ownershipFlow") {
+		return &task.OwnershipMismatchError{Ticket: "PAY-101", Workflow: "ownershipFlow"}
+	}
+	marker := "wf-owner:ownershipFlow:" + s.owner
+	if !containsLabel(s.state.labels, marker) {
+		s.state.labels = append(s.state.labels, marker)
+	}
+	return nil
+}
+
+func (s *ownershipCompositionTaskSystem) ClaimIfOwned(ctx context.Context, ticket task.TicketRef, workflowName string, _ config.RawValues) error {
+	s.state.mu.Lock()
+	defer s.state.mu.Unlock()
+	if s.state.assignee != s.owner {
+		return &task.OwnershipMismatchError{Ticket: ticket.Key, Workflow: workflowName}
+	}
+	ownerMarker := "wf-owner:" + workflowName + ":" + s.owner
+	claim := "wf:" + workflowName
+	for _, label := range s.state.labels {
+		if label == ownerMarker && containsLabel(s.state.labels, claim) {
+			return nil
+		}
+		if len(label) > len("wf:") && label[:len("wf:")] == "wf:" {
+			return &task.OwnershipMismatchError{Ticket: ticket.Key, Workflow: workflowName}
+		}
+	}
+	s.state.labels = append(s.state.labels, ownerMarker, claim)
+	return nil
+}
+
+func (s *ownershipCompositionTaskSystem) hasOwnerMarkerLocked(workflowName string) bool {
+	return containsLabel(s.state.labels, "wf-owner:"+workflowName+":"+s.owner)
+}
+
+func containsLabel(labels []string, want string) bool {
+	for _, label := range labels {
+		if label == want {
+			return true
+		}
+	}
+	return false
+}
+
+func TestExplicitLegacyBackfillPreservesClaimAndEnablesRoutingAndRecovery(t *testing.T) {
+	state := &ownershipCompositionState{assignee: "alice", labels: []string{"wf:ownershipFlow"}}
+	log := newEventLog()
+	sys := newOwnershipCompositionTaskSystem("alice", state, log)
+	reg := repoRegistryWith("payments", sys)
+	wf := linearWorkflow(false)
+	wf.Name = "ownershipFlow"
+	wf.TaskConfig = config.RawValues{"filters": map[string]any{"assignees": []any{"currentUser()"}}}
+	if err := reg.BindWorkflows([]*workflow.Workflow{&wf}); err != nil {
+		t.Fatal(err)
+	}
+	workflowReg := &workflow.Registry{}
+	workflowReg.Replace(&wf)
+	fr := newFakeRunner(log)
+	engine := newEngine(t, goworkflows.Dependencies{Repos: reg, Runner: fr, Harness: newFakeHarness(log)})
+	manager := &run.RunManager{Executor: engine, Runs: engine, Repos: reg, Workflows: workflowReg}
+	if err := manager.BackfillClaimOwner(context.Background(), "payments", "PAY-101", "ownershipFlow"); err != nil {
+		t.Fatalf("explicit backfill failed: %v", err)
+	}
+	state.mu.Lock()
+	labels := append([]string(nil), state.labels...)
+	state.mu.Unlock()
+	if !containsLabel(labels, "wf:ownershipFlow") || !containsLabel(labels, "wf-owner:ownershipFlow:alice") || len(labels) != 2 {
+		t.Fatalf("backfill labels = %v, want preserved claim plus one provenance marker", labels)
+	}
+	rp, _ := reg.Get("payments")
+	ticket, _ := sys.Poll(context.Background())
+	if _, err := router.ResolveWorkflow(rp, ticket[0]); err != nil {
+		t.Fatalf("backfilled claim did not route normally: %v", err)
+	}
+	specsFor := func(system task.System, work run.Work, w *workflow.Workflow) ([]task.MailboxSpec, error) {
+		return goworkflows.RenderMailboxSpecs(system, work, w)
+	}
+	if err := recoverpkg.FromTaskSystem(context.Background(), reg, fr, manager, specsFor); err != nil {
+		t.Fatalf("backfilled claim did not recover: %v", err)
+	}
+	if runs, err := engine.ListRuns(context.Background(), run.Filter{Repo: "payments", Workflow: "ownershipFlow", Ticket: "PAY-101"}); err != nil || len(runs) != 1 {
+		t.Fatalf("recovered runs = %v, %v; want one run", runs, err)
+	}
+}
+
+func TestIndependentRunDatabasesCannotTakeOverReassignedClaim(t *testing.T) {
+	state := &ownershipCompositionState{assignee: "alice"}
+	logA, logB := newEventLog(), newEventLog()
+	sysA := newOwnershipCompositionTaskSystem("alice", state, logA)
+	sysB := newOwnershipCompositionTaskSystem("bob", state, logB)
+	regA := repoRegistryWith("payments", sysA)
+	regB := repoRegistryWith("payments", sysB)
+	wf := linearWorkflow(false)
+	wf.Name = "ownershipFlow"
+	wf.TaskConfig = config.RawValues{"filters": map[string]any{
+		"assignees": []any{"currentUser()"},
+	}}
+	if err := regA.BindWorkflows([]*workflow.Workflow{&wf}); err != nil {
+		t.Fatal(err)
+	}
+	if err := regB.BindWorkflows([]*workflow.Workflow{&wf}); err != nil {
+		t.Fatal(err)
+	}
+
+	engineA := newEngine(t, goworkflows.Dependencies{
+		Repos: regA, Runner: newFakeRunner(logA), Harness: newFakeHarness(logA),
+	})
+	engineB := newEngine(t, goworkflows.Dependencies{
+		Repos: regB, Runner: newFakeRunner(logB), Harness: newFakeHarness(logB),
+	})
+	managerA := &run.RunManager{Executor: engineA, Runs: engineA}
+	managerB := &run.RunManager{Executor: engineB, Runs: engineB}
+
+	ticketA, err := sysA.Poll(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	rpA, _ := regA.Get("payments")
+	resolvedA, err := router.ResolveWorkflow(rpA, ticketA[0])
+	if err != nil {
+		t.Fatalf("owner A route failed: %v", err)
+	}
+	if err := managerA.EnsureRun(context.Background(), rpA, resolvedA, ticketA[0]); err != nil {
+		t.Fatalf("owner A EnsureRun failed: %v", err)
+	}
+	waitFor(t, 10*time.Second, func() bool {
+		current, err := engineA.GetRun(context.Background(), identity.NewRunID("payments", "ownershipFlow", "PAY-101"))
+		return err == nil && current.State != run.StateCompleted && current.State != run.StateCanceled
+	})
+
+	state.mu.Lock()
+	state.assignee = "bob"
+	state.mu.Unlock()
+	ticketB, err := sysB.Poll(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	rpB, _ := regB.Get("payments")
+	if _, err := router.ResolveWorkflow(rpB, ticketB[0]); !errors.Is(err, router.ErrClaimOwnerMismatch) {
+		t.Fatalf("owner B route error = %v, want claim-owner-mismatch", err)
+	}
+	if err := managerB.EnsureRun(context.Background(), rpB, &wf, ticketB[0]); !errors.Is(err, task.ErrOwnershipMismatch) {
+		t.Fatalf("owner B EnsureRun error = %v, want ownership mismatch", err)
+	}
+	if runs, err := engineB.ListRuns(context.Background(), run.Filter{Repo: "payments", Workflow: "ownershipFlow", Ticket: "PAY-101"}); err != nil || len(runs) != 0 {
+		t.Fatalf("owner B database runs = %v, %v; want none", runs, err)
+	}
+
+	// Reassignment is a current-owner mismatch for both servers. The
+	// original durable run remains in database A, but neither server calls
+	// EnsureRun to reconcile or create execution state after the mismatch.
+	ticketAAfterReassignment, _ := sysA.Poll(context.Background())
+	if _, err := router.ResolveWorkflow(rpA, ticketAAfterReassignment[0]); !errors.Is(err, router.ErrClaimOwnerMismatch) {
+		t.Fatalf("owner A route after reassignment = %v, want claim-owner-mismatch", err)
+	}
+	if err := managerA.EnsureRun(context.Background(), rpA, &wf, ticketAAfterReassignment[0]); !errors.Is(err, task.ErrOwnershipMismatch) {
+		t.Fatalf("owner A EnsureRun after reassignment = %v, want ownership mismatch", err)
+	}
+	if runs, err := engineA.ListRuns(context.Background(), run.Filter{Repo: "payments", Workflow: "ownershipFlow", Ticket: "PAY-101"}); err != nil || len(runs) != 1 {
+		t.Fatalf("owner A database runs after reassignment = %v, %v; want one existing run", runs, err)
+	}
+}