relay-flow 0.2.3-alpha → 0.2.5-alpha

package/internal/execution/temporal/recovery.go

@@ -0,0 +1,194 @@
+package temporal
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/rajpopat27/relay-flow/internal/identity"
+	enumspb "go.temporal.io/api/enums/v1"
+	"go.temporal.io/api/serviceerror"
+	workflowpb "go.temporal.io/api/workflow/v1"
+	workflowservice "go.temporal.io/api/workflowservice/v1"
+)
+
+// rebuildProjection reconstructs only relay-owned SQLite state. Temporal
+// histories remain untouched and task/runner/harness dependencies are never
+// called from this path.
+func (e *Engine) rebuildProjection(ctx context.Context) error {
+	if e.client == nil {
+		return fmt.Errorf("Temporal client is not connected")
+	}
+	// Querying an active workflow requires a workflow worker to replay its
+	// history and install the query handlers. LocalActivityWorkerOnly prevents
+	// this recovery worker from running any external activity.
+	recoveryWorker := e.newWorker(true)
+	if err := recoveryWorker.Start(); err != nil {
+		stopWorker(recoveryWorker)
+		return fmt.Errorf("start Temporal recovery worker: %w", err)
+	}
+	defer stopWorker(recoveryWorker)
+
+	executions, err := listWorkflowExecutions(ctx, func(callCtx context.Context, request *workflowservice.ListWorkflowExecutionsRequest) (*workflowservice.ListWorkflowExecutionsResponse, error) {
+		return e.client.ListWorkflow(callCtx, request)
+	})
+	if err != nil {
+		return fmt.Errorf("enumerate Temporal TicketWorkflow executions: %w", err)
+	}
+	selected := selectTemporalExecutions(executions, time.Now().UTC(), e.retention)
+	// The boolean records whether the selected Visibility history is a
+	// confirmed running execution. A retained closed history is deliberately
+	// false so claimed-parent reconciliation performs exact DescribeWorkflowExecution
+	// and can discover a newer active execution for the same Workflow ID.
+	visible := map[string]bool{}
+	for _, info := range selected {
+		visible[info.Execution.WorkflowId] = info.Status == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING
+		if info.Status == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING {
+			if err := e.restoreProjection(ctx, info); err != nil {
+				return fmt.Errorf("rebuild active Temporal workflow %s: %w", info.Execution.WorkflowId, err)
+			}
+			continue
+		}
+		if err := e.restoreProjection(ctx, info); err != nil {
+			return fmt.Errorf("rebuild Temporal workflow %s: %w", info.Execution.WorkflowId, err)
+		}
+	}
+	return e.reconcileClaimedParents(ctx, visible)
+}
+
+func shouldRestoreTemporalExecution(info *workflowpb.WorkflowExecutionInfo, now time.Time, retention time.Duration) bool {
+	if info == nil || info.Status == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING || info.CloseTime == nil {
+		return true
+	}
+	return !info.CloseTime.AsTime().Before(now.Add(-retention))
+}
+
+func selectTemporalExecutions(infos []*workflowpb.WorkflowExecutionInfo, now time.Time, retention time.Duration) []*workflowpb.WorkflowExecutionInfo {
+	selected := make(map[string]*workflowpb.WorkflowExecutionInfo)
+	for _, info := range infos {
+		if info == nil || info.Execution == nil || info.Type == nil || info.Type.Name != TicketWorkflowName || info.TaskQueue != TaskQueue || !shouldRestoreTemporalExecution(info, now, retention) {
+			continue
+		}
+		id := info.Execution.WorkflowId
+		current, ok := selected[id]
+		if !ok {
+			selected[id] = info
+			continue
+		}
+		other := info
+		if temporalExecutionPreferred(info, current) {
+			selected[id] = info
+			other = current
+		}
+		slog.Warn("multiple Temporal executions for Workflow ID; selecting one", "workflowID", id, "selectedRunID", selected[id].Execution.RunId, "otherRunID", other.Execution.RunId, "selectedStatus", selected[id].Status, "otherStatus", other.Status)
+	}
+	ids := make([]string, 0, len(selected))
+	for id := range selected {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	out := make([]*workflowpb.WorkflowExecutionInfo, 0, len(ids))
+	for _, id := range ids {
+		out = append(out, selected[id])
+	}
+	return out
+}
+
+func temporalExecutionPreferred(candidate, current *workflowpb.WorkflowExecutionInfo) bool {
+	candidateRunning := candidate.Status == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING
+	currentRunning := current.Status == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING
+	if candidateRunning != currentRunning {
+		return candidateRunning
+	}
+	if candidate.StartTime != nil && current.StartTime != nil && !candidate.StartTime.AsTime().Equal(current.StartTime.AsTime()) {
+		return candidate.StartTime.AsTime().After(current.StartTime.AsTime())
+	}
+	if candidate.CloseTime != nil && current.CloseTime != nil && !candidate.CloseTime.AsTime().Equal(current.CloseTime.AsTime()) {
+		return candidate.CloseTime.AsTime().After(current.CloseTime.AsTime())
+	}
+	return candidate.Execution.RunId > current.Execution.RunId
+}
+
+func listWorkflowExecutions(ctx context.Context, list func(context.Context, *workflowservice.ListWorkflowExecutionsRequest) (*workflowservice.ListWorkflowExecutionsResponse, error)) ([]*workflowpb.WorkflowExecutionInfo, error) {
+	var token []byte
+	var executions []*workflowpb.WorkflowExecutionInfo
+	for {
+		response, err := list(ctx, &workflowservice.ListWorkflowExecutionsRequest{
+			PageSize:      100,
+			Query:         fmt.Sprintf("WorkflowType = '%s' AND TaskQueue = '%s'", TicketWorkflowName, TaskQueue),
+			NextPageToken: token,
+		})
+		if err != nil {
+			return nil, err
+		}
+		if response == nil {
+			return nil, fmt.Errorf("empty response")
+		}
+		executions = append(executions, response.Executions...)
+		if len(response.NextPageToken) == 0 {
+			return executions, nil
+		}
+		token = response.NextPageToken
+	}
+}
+
+// reconcileClaimedParents closes the Visibility lag window without creating
+// anything. Task-system polling is read-only; an exact deterministic Workflow
+// ID is described and restored only when Temporal already owns that execution.
+func (e *Engine) reconcileClaimedParents(ctx context.Context, visible map[string]bool) error {
+	for _, rp := range e.deps.Repos.List() {
+		tickets, err := rp.TaskSystem.Poll(ctx)
+		if err != nil {
+			return fmt.Errorf("poll repo %q during Temporal recovery: %w", rp.Name, err)
+		}
+		for _, ticket := range tickets {
+			for _, claim := range ticket.WorkflowClaims {
+				if !strings.HasPrefix(claim, "wf:") {
+					continue
+				}
+				workflowName := strings.TrimPrefix(claim, "wf:")
+				var matched bool
+				for _, binding := range rp.Bindings() {
+					if binding.Workflow != nil && binding.Workflow.Name == workflowName {
+						matched = true
+						break
+					}
+				}
+				if !matched {
+					continue
+				}
+				expectedID := identity.NewRunID(rp.Name, workflowName, ticket.Key)
+				if visible[string(expectedID)] {
+					continue
+				}
+				info, err := e.client.DescribeWorkflowExecution(ctx, string(expectedID), "")
+				if err != nil {
+					var notFound *serviceerror.NotFound
+					if errors.As(err, &notFound) {
+						// This is the documented claim-before-run gap. Record it
+						// for operators, but leave creation to normal polling after
+						// recovery; this path must never start a replacement.
+						slog.Info("Temporal recovery missing claimed execution", "repo", rp.Name, "workflow", workflowName, "ticket", ticket.Key, "runID", string(expectedID))
+						continue
+					}
+					return fmt.Errorf("describe claimed Temporal workflow %s: %w", expectedID, err)
+				}
+				if info == nil || info.WorkflowExecutionInfo == nil || info.WorkflowExecutionInfo.Execution == nil ||
+					info.WorkflowExecutionInfo.Type == nil || info.WorkflowExecutionInfo.Type.Name != TicketWorkflowName || info.WorkflowExecutionInfo.TaskQueue != TaskQueue {
+					continue
+				}
+				if !shouldRestoreTemporalExecution(info.WorkflowExecutionInfo, time.Now().UTC(), e.retention) {
+					continue
+				}
+				if err := e.restoreProjection(ctx, info.WorkflowExecutionInfo); err != nil {
+					return fmt.Errorf("restore claimed Temporal workflow %s: %w", expectedID, err)
+				}
+			}
+		}
+	}
+	return nil
+}

package/internal/execution/temporal/recovery_runtime.go

@@ -0,0 +1,41 @@
+package temporal
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/rajpopat27/relay-flow/internal/execution/projection"
+	"github.com/rajpopat27/relay-flow/internal/run"
+	"github.com/rajpopat27/relay-flow/internal/runner"
+)
+
+// rediscoverMissingTerminals repairs only a missing derived terminal handle.
+// Runner adapters that support title discovery perform a read-only lookup;
+// recovery never creates a terminal or closes a healthy one.
+func (e *Engine) rediscoverMissingTerminals(ctx context.Context, start run.Start, state RunStateSnapshot) error {
+	discoverer, ok := e.deps.Runner.(runner.TerminalDiscoverer)
+	if !ok {
+		return nil
+	}
+	spec := runner.RunSpec{RunID: start.ID, RepoName: start.Repo, RepoPath: start.RepoPath, TicketKey: start.Ticket.Key}
+	for _, binding := range state.RuntimeBindings {
+		if binding.Node == "" || binding.Node != state.Run.CurrentNode || binding.NodeVisitID == "" || binding.TerminalID != "" {
+			continue
+		}
+		title := start.Ticket.Key + ":" + binding.Node
+		terminal, found, err := discoverer.DiscoverTerminal(ctx, spec, title)
+		if err != nil {
+			return fmt.Errorf("discover terminal %q during Temporal recovery: %w", title, err)
+		}
+		if !found || terminal.ID == "" {
+			continue
+		}
+		if err := e.runs.UpdateNodeRuntime(ctx, projection.NodeRuntime{
+			RunID: start.ID, Node: binding.Node, TerminalID: terminal.ID,
+			SessionID: binding.SessionID, NodeVisitID: binding.NodeVisitID,
+		}); err != nil {
+			return fmt.Errorf("restore terminal binding %q: %w", title, err)
+		}
+	}
+	return nil
+}

package/internal/execution/temporal/recovery_test.go

@@ -0,0 +1,102 @@
+package temporal
+
+import (
+	"bytes"
+	"context"
+	"testing"
+	"time"
+
+	commonpb "go.temporal.io/api/common/v1"
+	enumspb "go.temporal.io/api/enums/v1"
+	workflowpb "go.temporal.io/api/workflow/v1"
+	workflowservice "go.temporal.io/api/workflowservice/v1"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestRecoveryWorkerOptionsAreWorkflowOnly(t *testing.T) {
+	recovery := workerOptions(nil, true)
+	if !recovery.LocalActivityWorkerOnly {
+		t.Fatal("recovery worker enables non-local activities")
+	}
+	if recovery.MaxConcurrentWorkflowTaskExecutionSize != 10 || recovery.MaxConcurrentWorkflowTaskPollers != 2 {
+		t.Fatalf("recovery workflow options = %+v", recovery)
+	}
+	normal := workerOptions(nil, false)
+	if normal.LocalActivityWorkerOnly {
+		t.Fatal("normal worker was restricted to local activities")
+	}
+}
+
+func TestShouldRestoreTemporalExecutionHonorsLocalRetention(t *testing.T) {
+	now := time.Date(2026, 9, 4, 0, 0, 0, 0, time.UTC)
+	retention := 30 * 24 * time.Hour
+	if !shouldRestoreTemporalExecution(&workflowpb.WorkflowExecutionInfo{Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING}, now, retention) {
+		t.Fatal("active execution was filtered by local retention")
+	}
+	recent := timestamppb.New(now.Add(-retention + time.Minute))
+	if !shouldRestoreTemporalExecution(&workflowpb.WorkflowExecutionInfo{Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, CloseTime: recent}, now, retention) {
+		t.Fatal("recent closed execution was filtered")
+	}
+	old := timestamppb.New(now.Add(-retention - time.Minute))
+	if shouldRestoreTemporalExecution(&workflowpb.WorkflowExecutionInfo{Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, CloseTime: old}, now, retention) {
+		t.Fatal("expired closed execution was restored")
+	}
+}
+
+func TestSelectTemporalExecutionsKeepsOneCurrentHistoryPerWorkflowID(t *testing.T) {
+	now := time.Date(2026, 9, 4, 0, 0, 0, 0, time.UTC)
+	info := func(runID string, status enumspb.WorkflowExecutionStatus, started time.Time) *workflowpb.WorkflowExecutionInfo {
+		return &workflowpb.WorkflowExecutionInfo{
+			Execution: &commonpb.WorkflowExecution{WorkflowId: "same-id", RunId: runID},
+			Type:      &commonpb.WorkflowType{Name: TicketWorkflowName}, TaskQueue: TaskQueue,
+			Status: status, StartTime: timestamppb.New(started),
+		}
+	}
+	selected := selectTemporalExecutions([]*workflowpb.WorkflowExecutionInfo{
+		info("old", enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, now.Add(-time.Hour)),
+		info("current", enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, now),
+		{
+			Execution: &commonpb.WorkflowExecution{WorkflowId: "other", RunId: "other"},
+			Type:      &commonpb.WorkflowType{Name: "OtherWorkflow"}, TaskQueue: TaskQueue,
+			Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, StartTime: timestamppb.New(now),
+		},
+	}, now, 30*24*time.Hour)
+	if len(selected) != 1 || selected[0].Execution.RunId != "current" {
+		t.Fatalf("selected histories = %#v", selected)
+	}
+}
+
+func TestListWorkflowExecutionsPaginatesWithExactFilter(t *testing.T) {
+	var requests []*workflowservice.ListWorkflowExecutionsRequest
+	list := func(_ context.Context, request *workflowservice.ListWorkflowExecutionsRequest) (*workflowservice.ListWorkflowExecutionsResponse, error) {
+		requests = append(requests, request)
+		if len(requests) == 1 {
+			return &workflowservice.ListWorkflowExecutionsResponse{
+				Executions:    []*workflowpb.WorkflowExecutionInfo{{Execution: &commonpb.WorkflowExecution{WorkflowId: "first"}}},
+				NextPageToken: []byte("next-page"),
+			}, nil
+		}
+		return &workflowservice.ListWorkflowExecutionsResponse{
+			Executions: []*workflowpb.WorkflowExecutionInfo{{Execution: &commonpb.WorkflowExecution{WorkflowId: "second"}}},
+		}, nil
+	}
+	executions, err := listWorkflowExecutions(context.Background(), list)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(executions) != 2 || executions[0].Execution.WorkflowId != "first" || executions[1].Execution.WorkflowId != "second" {
+		t.Fatalf("executions = %#v", executions)
+	}
+	if len(requests) != 2 || requests[0].PageSize != 100 || requests[1].PageSize != 100 {
+		t.Fatalf("pagination requests = %#v", requests)
+	}
+	wantQuery := "WorkflowType = '" + TicketWorkflowName + "' AND TaskQueue = '" + TaskQueue + "'"
+	for i, request := range requests {
+		if request.Query != wantQuery {
+			t.Fatalf("request %d query = %q, want %q", i, request.Query, wantQuery)
+		}
+	}
+	if !bytes.Equal(requests[1].NextPageToken, []byte("next-page")) {
+		t.Fatalf("second page token = %q", requests[1].NextPageToken)
+	}
+}

package/internal/execution/temporal/snapshot_restart_test.go

@@ -0,0 +1,72 @@
+package temporal
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/rajpopat27/relay-flow/internal/execution/projection"
+	"github.com/rajpopat27/relay-flow/internal/identity"
+	"github.com/rajpopat27/relay-flow/internal/repo"
+	"github.com/rajpopat27/relay-flow/internal/run"
+	"github.com/rajpopat27/relay-flow/internal/task"
+)
+
+func TestTemporalWorkflowSnapshotSurvivesWorkerRestart(t *testing.T) {
+	if os.Getenv("RELAY_FLOW_TEMPORAL_LIVE") != "1" {
+		t.Skip("set RELAY_FLOW_TEMPORAL_LIVE=1 to run Temporal snapshot restart coverage")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	namespace := "relay-flow-snapshot-" + string(identity.NewNodeVisitID())[:12]
+	if err := ensureSpikeNamespace(ctx, "localhost:7233", namespace); err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(t.TempDir(), "state.db")
+	if err := projection.InitDatabaseWithIdentity(path, projection.ExecutorIdentity{ExecutorPlugin: "temporal", TemporalAddress: "localhost:7233", TemporalNamespace: namespace}); err != nil {
+		t.Fatal(err)
+	}
+	sys := &lagTaskSystem{}
+	registry := repo.NewRegistry()
+	wf := lagWorkflow()
+	registry.Replace(&repo.Repo{Name: "repo", Path: "/repo", TaskSystem: sys, Workflows: []repo.WorkflowBinding{{Workflow: &wf}}})
+	deps := Dependencies{Repos: registry, Runner: &lagRunner{}, Harness: &lagHarness{}, TaskSystem: "lag-task", TemporalAddress: "localhost:7233", TemporalNamespace: namespace}
+	engine, err := New(path, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := engine.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+	start := run.Start{ID: identity.NewRunID("repo", wf.Name, "SNAPSHOT-1"), Repo: "repo", RepoPath: "/repo", Workflow: wf, Ticket: task.TicketRef{ID: "SNAPSHOT-1", Key: "SNAPSHOT-1", Title: "Snapshot"}}
+	if created, err := engine.EnsureRun(ctx, start); err != nil || !created {
+		t.Fatalf("EnsureRun = %v, %v", created, err)
+	}
+	waitForLagState(t, ctx, engine, start.ID, run.StateWaiting)
+	before, err := engine.workflowFromHistory(ctx, start.ID, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := engine.Shutdown(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	restarted, err := New(path, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := restarted.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+	defer restarted.Shutdown(context.Background())
+	after, err := restarted.workflowFromHistory(ctx, start.ID, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if after.Name != before.Name || after.Nodes["work"].Description != before.Nodes["work"].Description || after.Nodes["work"].Agent != before.Nodes["work"].Agent {
+		t.Fatalf("snapshot changed across worker restart: before=%+v after=%+v", before, after)
+	}
+	waitForLagState(t, ctx, restarted, start.ID, run.StateWaiting)
+}