relay-flow 0.2.0-alpha → 0.2.2-alpha

package/internal/repo/repo.go

@@ -32,12 +32,23 @@ type Repo struct {
 	TaskConfig config.RawValues
 	TaskSystem task.System
 	Workflows  []WorkflowBinding
+	bindingsMu sync.RWMutex
 }
 
 func (r *Repo) Info() Info {
 	return Info{Name: r.Name, Path: r.Path, TaskConfig: r.TaskConfig}
 }
 
+// Bindings returns a snapshot of the repo's currently published workflow
+// bindings. WorkflowBinding values are immutable after publication.
+func (r *Repo) Bindings() []WorkflowBinding {
+	r.bindingsMu.RLock()
+	bindings := make([]WorkflowBinding, len(r.Workflows))
+	copy(bindings, r.Workflows)
+	r.bindingsMu.RUnlock()
+	return bindings
+}
+
 // Registry is the in-memory repo set.
 type Registry struct {
 	mu   sync.RWMutex
@@ -113,7 +124,9 @@ func (r *Registry) BindWorkflows(workflows []*workflow.Workflow) error {
 			next = append(next, WorkflowBinding{Workflow: b.wf, Match: b.match})
 		}
 		sort.Slice(next, func(i, j int) bool { return next[i].Workflow.Name < next[j].Workflow.Name })
+		rp.bindingsMu.Lock()
 		rp.Workflows = next
+		rp.bindingsMu.Unlock()
 	}
 	return nil
 }

package/internal/repo/service.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/rajpopat27/relay-flow/internal/config"
+	"github.com/rajpopat27/relay-flow/internal/harness"
 	"github.com/rajpopat27/relay-flow/internal/runner"
 	"github.com/rajpopat27/relay-flow/internal/task"
 )
@@ -26,6 +27,7 @@ type ServiceConfig struct {
 	ConfigPath string
 	TaskPlugin string
 	Runner     runner.Runner
+	Harness    harness.Harness
 	Active     ActiveRuns
 	Workflows  WorkflowRefs
 }
@@ -34,6 +36,7 @@ type Service struct {
 	cfgPath    string
 	taskPlugin string
 	runner     runner.Runner
+	harness    harness.Harness
 	active     ActiveRuns
 	workflows  WorkflowRefs
 	reg        *Registry
@@ -51,6 +54,7 @@ func NewServiceWithRegistry(cfg ServiceConfig, reg *Registry) *Service {
 		cfgPath:    cfg.ConfigPath,
 		taskPlugin: cfg.TaskPlugin,
 		runner:     cfg.Runner,
+		harness:    cfg.Harness,
 		active:     cfg.Active,
 		workflows:  cfg.Workflows,
 		reg:        reg,
@@ -141,6 +145,12 @@ func (s *Service) Register(ctx context.Context, input RegisterInput) (Info, erro
 	if err != nil {
 		return Info{}, fmt.Errorf("repo %q: task system connectivity: %w", input.Name, err)
 	}
+	if s.harness == nil {
+		return Info{}, fmt.Errorf("repo %q: harness is not configured", input.Name)
+	}
+	if err := s.harness.SetupRepo(ctx, input.Path); err != nil {
+		return Info{}, fmt.Errorf("repo %q: harness setup: %w", input.Name, err)
+	}
 	// Persist atomically, then update the in-memory registry.
 	r := config.Repo{Path: input.Path, TaskConfig: input.TaskConfig}
 	if cfg.Repos == nil {

package/internal/repo/service_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/rajpopat27/relay-flow/internal/config"
+	"github.com/rajpopat27/relay-flow/internal/harness"
 	"github.com/rajpopat27/relay-flow/internal/repo"
 	"github.com/rajpopat27/relay-flow/internal/runner"
 	"github.com/rajpopat27/relay-flow/internal/task"
@@ -29,10 +30,10 @@ func (f *fakeRunnerDiscovery) ValidateRepo(_ context.Context, _, _ string) error
 func (f *fakeRunnerDiscovery) EnsureEnvironment(context.Context, runner.RunSpec) (runner.Environment, error) {
 	return runner.Environment{}, nil
 }
-func (f *fakeRunnerDiscovery) FindTerminal(context.Context, runner.Environment, string) (runner.Terminal, bool, error) {
-	return runner.Terminal{}, false, nil
+func (f *fakeRunnerDiscovery) SetEnvironmentStatus(context.Context, runner.Environment, string) error {
+	return nil
 }
-func (f *fakeRunnerDiscovery) InspectTerminal(context.Context, runner.Terminal) (runner.Terminal, bool, error) {
+func (f *fakeRunnerDiscovery) FindTerminal(context.Context, runner.Terminal) (runner.Terminal, bool, error) {
 	return runner.Terminal{}, false, nil
 }
 func (f *fakeRunnerDiscovery) SendTerminal(context.Context, runner.Terminal, string) error {
@@ -42,12 +43,32 @@ func (f *fakeRunnerDiscovery) CreateTerminal(context.Context, runner.Environment
 	return runner.Terminal{}, nil
 }
 func (f *fakeRunnerDiscovery) CloseTerminal(context.Context, runner.Terminal) error { return nil }
-func (f *fakeRunnerDiscovery) EnsureTerminal(context.Context, runner.Environment, string, runner.Command) (runner.Terminal, error) {
+func (f *fakeRunnerDiscovery) EnsureTerminal(context.Context, runner.Environment, runner.Terminal, string, runner.Command) (runner.Terminal, error) {
 	return runner.Terminal{}, nil
 }
 func (f *fakeRunnerDiscovery) CloseTerminals(context.Context, runner.RunSpec) error { return nil }
 func (f *fakeRunnerDiscovery) CleanupRun(context.Context, runner.RunSpec) error     { return nil }
 
+type fakeHarness struct {
+	setupPaths []string
+	setupErr   error
+}
+
+func (f *fakeHarness) SetupRepo(_ context.Context, path string) error {
+	f.setupPaths = append(f.setupPaths, path)
+	return f.setupErr
+}
+func (*fakeHarness) ValidateAgent(context.Context, string, string) error { return nil }
+func (*fakeHarness) FindSession(context.Context, string, string) (harness.Session, bool, error) {
+	return harness.Session{}, false, nil
+}
+func (*fakeHarness) RenderPrompt(harness.PromptKind, harness.PromptData, string) (string, error) {
+	return "", nil
+}
+func (*fakeHarness) BuildCommand(harness.LaunchSpec) (runner.Command, error) {
+	return runner.Command{}, nil
+}
+
 // fakeActiveRuns / fakeWorkflowRefs implement the consumer interfaces.
 type fakeActiveRuns struct{ repos map[string]bool }
 
@@ -102,6 +123,7 @@ type serviceFixture struct {
 	cfgPath string
 	active  *fakeActiveRuns
 	wfRefs  *fakeWorkflowRefs
+	harness *fakeHarness
 }
 
 func newServiceFixture(t *testing.T, rn runner.Runner) serviceFixture {
@@ -118,14 +140,16 @@ func newServiceFixture(t *testing.T, rn runner.Runner) serviceFixture {
 	}
 	active := &fakeActiveRuns{repos: map[string]bool{}}
 	wfRefs := &fakeWorkflowRefs{refs: map[string]bool{}}
+	hrn := &fakeHarness{}
 	svc := repo.NewService(repo.ServiceConfig{
 		ConfigPath: cfgPath,
 		TaskPlugin: "testjira",
 		Runner:     rn,
+		Harness:    hrn,
 		Active:     active,
 		Workflows:  wfRefs,
 	})
-	return serviceFixture{svc: svc, cfgPath: cfgPath, active: active, wfRefs: wfRefs}
+	return serviceFixture{svc: svc, cfgPath: cfgPath, active: active, wfRefs: wfRefs, harness: hrn}
 }
 
 func TestDiscoverDelegatesToRunner(t *testing.T) {
@@ -222,7 +246,7 @@ func TestRegisterValidatesTaskConnectivity(t *testing.T) {
 	}
 	svc := repo.NewService(repo.ServiceConfig{
 		ConfigPath: cfgPath, TaskPlugin: "failingtask",
-		Runner: &fakeRunnerDiscovery{}, Active: &fakeActiveRuns{repos: map[string]bool{}}, Workflows: &fakeWorkflowRefs{refs: map[string]bool{}},
+		Runner: &fakeRunnerDiscovery{}, Harness: &fakeHarness{}, Active: &fakeActiveRuns{repos: map[string]bool{}}, Workflows: &fakeWorkflowRefs{refs: map[string]bool{}},
 	})
 	_, err := svc.Register(context.Background(), repo.RegisterInput{Name: "payments", Path: "/srv/payments", TaskConfig: config.RawValues{"project": "PAY"}})
 	if err == nil {
@@ -256,6 +280,31 @@ func TestRegisterAtomicallyPersists(t *testing.T) {
 	}
 }
 
+func TestRegisterSetsUpHarnessRepo(t *testing.T) {
+	fx := newServiceFixture(t, &fakeRunnerDiscovery{})
+	if _, err := fx.svc.Register(context.Background(), repo.RegisterInput{Name: "payments", Path: "/srv/payments", TaskConfig: config.RawValues{"project": "PAY", "component": "api"}}); err != nil {
+		t.Fatal(err)
+	}
+	if len(fx.harness.setupPaths) != 1 || fx.harness.setupPaths[0] != "/srv/payments" {
+		t.Fatalf("SetupRepo paths = %v", fx.harness.setupPaths)
+	}
+}
+
+func TestRegisterHarnessSetupFailureDoesNotPersist(t *testing.T) {
+	fx := newServiceFixture(t, &fakeRunnerDiscovery{})
+	fx.harness.setupErr = errors.New("setup failed")
+	if _, err := fx.svc.Register(context.Background(), repo.RegisterInput{Name: "payments", Path: "/srv/payments", TaskConfig: config.RawValues{"project": "PAY", "component": "api"}}); err == nil {
+		t.Fatal("registration succeeded despite harness setup failure")
+	}
+	cfg, err := config.LoadMachine(fx.cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cfg.Repos) != 0 {
+		t.Fatalf("repos persisted after setup failure: %v", cfg.Repos)
+	}
+}
+
 func TestRemoveStopsPollerAndRemoves(t *testing.T) {
 	fx := newServiceFixture(t, &fakeRunnerDiscovery{})
 	if _, err := fx.svc.Register(context.Background(), repo.RegisterInput{Name: "payments", Path: "/srv/payments", TaskConfig: config.RawValues{"project": "PAY", "component": "api"}}); err != nil {

package/internal/router/router.go

@@ -52,9 +52,10 @@ func ResolveWorkflow(registered *repo.Repo, ticket task.Ticket) (*workflow.Workf
 	if len(claims) > 1 {
 		return nil, &InvalidClaimError{Ticket: ticket.Key, Workflow: strings.Join(claims, ","), Repo: registered.Name}
 	}
+	bindings := registered.Bindings()
 	if len(claims) == 1 {
 		name := strings.TrimPrefix(claims[0], claimPrefix)
-		for _, b := range registered.Workflows {
+		for _, b := range bindings {
 			if b.Workflow.Name == name {
 				return b.Workflow, nil
 			}
@@ -62,7 +63,7 @@ func ResolveWorkflow(registered *repo.Repo, ticket task.Ticket) (*workflow.Workf
 		return nil, &InvalidClaimError{Ticket: ticket.Key, Workflow: name, Repo: registered.Name}
 	}
 	var matched []*workflow.Workflow
-	for _, b := range registered.Workflows {
+	for _, b := range bindings {
 		if b.Match != nil && b.Match(ticket) {
 			matched = append(matched, b.Workflow)
 		}

package/internal/router/router_test.go

@@ -1,9 +1,14 @@
 package router_test
 
 import (
+	"context"
 	"errors"
+	"fmt"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"github.com/rajpopat27/relay-flow/internal/config"
 	"github.com/rajpopat27/relay-flow/internal/repo"
 	"github.com/rajpopat27/relay-flow/internal/router"
 	"github.com/rajpopat27/relay-flow/internal/task"
@@ -142,3 +147,85 @@ func TestMultipleMatchesAmbiguous(t *testing.T) {
 		t.Fatalf("ticket mutated on ambiguity: %+v", ticket)
 	}
 }
+
+type routingTaskSystem struct{ task.System }
+
+func (routingTaskSystem) Poll(context.Context) ([]task.Ticket, error) {
+	return []task.Ticket{{Key: "PAY-101"}}, nil
+}
+
+func (routingTaskSystem) CompileFilter(config.RawValues) (func(task.Ticket) bool, error) {
+	return func(task.Ticket) bool { return true }, nil
+}
+
+func TestPollerRoutesWhileBindingsAreReplaced(t *testing.T) {
+	registry := repo.NewRegistry()
+	registered := &repo.Repo{Name: "payments", TaskSystem: routingTaskSystem{}}
+	registry.Replace(registered)
+
+	first, second := wf("firstFlow"), wf("secondFlow")
+	if err := registry.BindWorkflows([]*workflow.Workflow{first}); err != nil {
+		t.Fatal(err)
+	}
+
+	var routed atomic.Int64
+	failures := make(chan error, 1)
+	poller := &repo.RepoPoller{
+		Repo:     registered,
+		Interval: time.Microsecond,
+		Handle: func(_ context.Context, registered *repo.Repo, tickets []task.Ticket) {
+			for _, ticket := range tickets {
+				resolved, err := router.ResolveWorkflow(registered, ticket)
+				if err != nil {
+					select {
+					case failures <- err:
+					default:
+					}
+					continue
+				}
+				if resolved != first && resolved != second {
+					select {
+					case failures <- fmt.Errorf("resolved unexpected workflow %q", resolved.Name):
+					default:
+					}
+				}
+				routed.Add(1)
+			}
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		poller.Run(ctx)
+		close(done)
+	}()
+
+	for i := 0; i < 1_000; i++ {
+		bindings := []*workflow.Workflow{first}
+		if i%2 == 1 {
+			bindings = []*workflow.Workflow{second}
+		}
+		if err := registry.BindWorkflows(bindings); err != nil {
+			cancel()
+			<-done
+			t.Fatal(err)
+		}
+	}
+
+	deadline := time.Now().Add(time.Second)
+	for routed.Load() == 0 && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	cancel()
+	<-done
+
+	select {
+	case err := <-failures:
+		t.Fatalf("routing during binding replacement failed: %v", err)
+	default:
+	}
+	if routed.Load() == 0 {
+		t.Fatal("poller did not route while bindings were replaced")
+	}
+}

package/internal/run/manager.go

@@ -47,6 +47,19 @@ func (m *RunManager) EnsureRun(ctx context.Context, rp *repo.Repo, wf *workflow.
 			break
 		}
 	}
+	existingRun := false
+	if claimed && m.Runs != nil {
+		existing, err := m.Runs.ListRuns(ctx, Filter{Repo: rp.Name, Workflow: wf.Name, Ticket: ticket.Key})
+		if err != nil {
+			return fmt.Errorf("check existing run %s: %w", id, err)
+		}
+		for _, candidate := range existing {
+			if candidate.ID == id {
+				existingRun = true
+				break
+			}
+		}
+	}
 	if !claimed {
 		if err := rp.TaskSystem.Claim(ctx, ticket.Ref(), wf.Name); err != nil {
 			slog.Info("ensure-run outcome",
@@ -54,7 +67,7 @@ func (m *RunManager) EnsureRun(ctx context.Context, rp *repo.Repo, wf *workflow.
 				"outcome", "error", "stage", "claim", "error", err)
 			return fmt.Errorf("claim %s for workflow %s: %w", ticket.Key, wf.Name, err)
 		}
-	} else {
+	} else if !existingRun {
 		// Claimed but possibly missing its run (claim-before-run crash gap or
 		// retention cleanup): never recreate a canceled run.
 		marked, err := rp.TaskSystem.HasComment(ctx, task.Target{Parent: ticket.Ref()}, CancellationMarker(id))

package/internal/run/run.go

@@ -63,10 +63,12 @@ type NodeWork struct {
 type CommentWork struct {
 	// RunID is the enclosing durable run; the comment activity uses it to
 	// attribute its 9.3 transition log lines (ticket/runID/node attrs).
-	RunID  ID
-	Item   task.Target
-	Body   string
-	Marker string
+	RunID    ID
+	Item     task.Target
+	Body     string
+	TextKind task.TextKind
+	TextData task.TextData
+	Marker   string
 }
 
 // RetryStatus describes the currently pending durable activity retry without

package/internal/run/run_manager_test.go

@@ -81,6 +81,8 @@ func (e *fakeExecutor) CancelRun(context.Context, run.ID, string) error { return
 // fakeQueries answers run lookups.
 type fakeQueries struct {
 	byTicket map[string]run.Run
+	list     []run.Run
+	listErr  error
 }
 
 func (q *fakeQueries) GetRun(_ context.Context, id run.ID) (run.Run, error) {
@@ -99,7 +101,9 @@ func (q *fakeQueries) FindRunByTicket(_ context.Context, ticket string) (run.Run
 	return run.Run{}, errors.New("not found")
 }
 
-func (q *fakeQueries) ListRuns(context.Context, run.Filter) ([]run.Run, error) { return nil, nil }
+func (q *fakeQueries) ListRuns(context.Context, run.Filter) ([]run.Run, error) {
+	return append([]run.Run(nil), q.list...), q.listErr
+}
 func (q *fakeQueries) HasActiveWorkflow(context.Context, string) (bool, error) { return false, nil }
 func (q *fakeQueries) HasActiveRepo(context.Context, string) (bool, error)     { return false, nil }
 
@@ -171,6 +175,22 @@ func TestClaimedTicketSkipsClaiming(t *testing.T) {
 	}
 }
 
+func TestExistingRunSkipsJiraCalls(t *testing.T) {
+	log := newEventLog()
+	sys := &recordingSystem{log: log, hasCancel: true}
+	id := identity.NewRunID("payments", "basicFlow", "PAY-101")
+	exec := &fakeExecutor{log: log}
+	m := &run.RunManager{Executor: exec, Runs: &fakeQueries{list: []run.Run{{ID: id}}}}
+	ticket := task.Ticket{ID: "1", Key: "PAY-101", WorkflowClaims: []string{"wf:basicFlow"}}
+
+	if err := m.EnsureRun(context.Background(), testRepo(sys), testWorkflow("basicFlow"), ticket); err != nil {
+		t.Fatal(err)
+	}
+	if sys.hasCommentN != 0 || len(log.all()) != 1 || len(exec.ensures) != 1 {
+		t.Fatalf("existing run caused external work: comments=%d events=%v ensures=%d", sys.hasCommentN, log.all(), len(exec.ensures))
+	}
+}
+
 func TestClaimFailurePreventsRunCreation(t *testing.T) {
 	log := newEventLog()
 	sys := &recordingSystem{log: log, claimErr: errors.New("jira down")}

package/internal/runner/contract_test.go

@@ -2,6 +2,7 @@ package runner_test
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 
@@ -20,9 +21,13 @@ type fakeRunner struct {
 	closed     []string
 	cleaned    []string
 	ensureN    int
+	findN      int
+	createN    int
 	ensureEnvN int
 }
 
+var _ runner.Runner = (*fakeRunner)(nil)
+
 func newFakeRunner() *fakeRunner {
 	return &fakeRunner{
 		envs:      map[string]runner.Environment{},
@@ -57,16 +62,12 @@ func (f *fakeRunner) EnsureEnvironment(_ context.Context, s runner.RunSpec) (run
 	return env, nil
 }
 
-func (f *fakeRunner) FindTerminal(_ context.Context, env runner.Environment, title string) (runner.Terminal, bool, error) {
-	key := env.ID + "/" + title
-	term, ok := f.terminals[key]
-	if !ok || f.dead[key] {
-		// Only a live usable terminal is returned.
-		return runner.Terminal{}, false, nil
-	}
-	return term, true, nil
+func (*fakeRunner) SetEnvironmentStatus(context.Context, runner.Environment, string) error {
+	return nil
 }
-func (f *fakeRunner) InspectTerminal(_ context.Context, term runner.Terminal) (runner.Terminal, bool, error) {
+
+func (f *fakeRunner) FindTerminal(_ context.Context, term runner.Terminal) (runner.Terminal, bool, error) {
+	f.findN++
 	for key, current := range f.terminals {
 		if current.ID == term.ID && !f.dead[key] {
 			return current, true, nil
@@ -75,8 +76,13 @@ func (f *fakeRunner) InspectTerminal(_ context.Context, term runner.Terminal) (r
 	return runner.Terminal{}, false, nil
 }
 func (f *fakeRunner) SendTerminal(context.Context, runner.Terminal, string) error { return nil }
-func (f *fakeRunner) CreateTerminal(ctx context.Context, env runner.Environment, title string, command runner.Command) (runner.Terminal, error) {
-	return f.EnsureTerminal(ctx, env, title, command)
+func (f *fakeRunner) CreateTerminal(_ context.Context, env runner.Environment, title string, _ runner.Command) (runner.Terminal, error) {
+	f.createN++
+	key := env.ID + "/" + title
+	term := runner.Terminal{ID: fmt.Sprintf("term-%d-%s", f.createN, key), Title: title}
+	f.terminals[key] = term
+	delete(f.dead, key)
+	return term, nil
 }
 
 func (f *fakeRunner) CloseTerminal(_ context.Context, term runner.Terminal) error {
@@ -89,16 +95,14 @@ func (f *fakeRunner) CloseTerminal(_ context.Context, term runner.Terminal) erro
 	return nil
 }
 
-func (f *fakeRunner) EnsureTerminal(_ context.Context, env runner.Environment, title string, _ runner.Command) (runner.Terminal, error) {
+func (f *fakeRunner) EnsureTerminal(ctx context.Context, env runner.Environment, stored runner.Terminal, title string, command runner.Command) (runner.Terminal, error) {
 	f.ensureN++
-	key := env.ID + "/" + title
-	if term, ok := f.terminals[key]; ok && !f.dead[key] {
+	if term, ok, err := f.FindTerminal(ctx, stored); err != nil {
+		return runner.Terminal{}, err
+	} else if ok {
 		return term, nil
 	}
-	term := runner.Terminal{ID: "term-" + key, Title: title}
-	f.terminals[key] = term
-	delete(f.dead, key)
-	return term, nil
+	return f.CreateTerminal(ctx, env, title, command)
 }
 
 func (f *fakeRunner) CloseTerminals(_ context.Context, s runner.RunSpec) error {
@@ -128,7 +132,7 @@ func TestTerminalTitleIsTicketColonNode(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	term, err := f.EnsureTerminal(ctx, env, "PAY-101:coding", runner.Command{Executable: "opencode"})
+	term, err := f.EnsureTerminal(ctx, env, runner.Terminal{}, "PAY-101:coding", runner.Command{Executable: "opencode"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -148,16 +152,16 @@ func TestFindTerminalReturnsOnlyLiveUsable(t *testing.T) {
 	f := newFakeRunner()
 	ctx := context.Background()
 	env, _ := f.EnsureEnvironment(ctx, spec())
-	term, _ := f.EnsureTerminal(ctx, env, "PAY-101:coding", runner.Command{})
+	term, _ := f.EnsureTerminal(ctx, env, runner.Terminal{}, "PAY-101:coding", runner.Command{})
 
-	got, ok, err := f.FindTerminal(ctx, env, "PAY-101:coding")
+	got, ok, err := f.FindTerminal(ctx, term)
 	if err != nil || !ok || got.ID != term.ID {
 		t.Fatalf("FindTerminal = %v,%v,%v; want live terminal", got, ok, err)
 	}
 
 	// A stale/dead record is treated as absent.
 	f.dead[env.ID+"/PAY-101:coding"] = true
-	_, ok, err = f.FindTerminal(ctx, env, "PAY-101:coding")
+	_, ok, err = f.FindTerminal(ctx, term)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -171,7 +175,7 @@ func TestCloseTerminalsPreservesEnvironment(t *testing.T) {
 	ctx := context.Background()
 	sp := spec()
 	env, _ := f.EnsureEnvironment(ctx, sp)
-	_, _ = f.EnsureTerminal(ctx, env, "PAY-101:coding", runner.Command{})
+	_, _ = f.EnsureTerminal(ctx, env, runner.Terminal{}, "PAY-101:coding", runner.Command{})
 
 	if err := f.CloseTerminals(ctx, sp); err != nil {
 		t.Fatal(err)
@@ -189,7 +193,7 @@ func TestCleanupRunRemovesAllRunResources(t *testing.T) {
 	ctx := context.Background()
 	sp := spec()
 	env, _ := f.EnsureEnvironment(ctx, sp)
-	_, _ = f.EnsureTerminal(ctx, env, "PAY-101:coding", runner.Command{})
+	_, _ = f.EnsureTerminal(ctx, env, runner.Terminal{}, "PAY-101:coding", runner.Command{})
 
 	if err := f.CleanupRun(ctx, sp); err != nil {
 		t.Fatal(err)
@@ -213,9 +217,43 @@ func TestEnsureIdempotent(t *testing.T) {
 		t.Fatalf("EnsureEnvironment not idempotent: %v vs %v", env1, env2)
 	}
 
-	term1, _ := f.EnsureTerminal(ctx, env1, "PAY-101:coding", runner.Command{})
-	term2, _ := f.EnsureTerminal(ctx, env1, "PAY-101:coding", runner.Command{})
+	term1, _ := f.EnsureTerminal(ctx, env1, runner.Terminal{}, "PAY-101:coding", runner.Command{})
+	term2, _ := f.EnsureTerminal(ctx, env1, term1, "PAY-101:coding", runner.Command{})
 	if term1 != term2 {
 		t.Fatalf("EnsureTerminal not idempotent: %v vs %v", term1, term2)
 	}
+	if f.findN != 2 || f.createN != 1 {
+		t.Fatalf("EnsureTerminal calls: find=%d create=%d, want find=2 create=1", f.findN, f.createN)
+	}
+}
+
+func TestEnsureTerminalReplacesUnavailableStoredTerminal(t *testing.T) {
+	f := newFakeRunner()
+	ctx := context.Background()
+	env, _ := f.EnsureEnvironment(ctx, spec())
+	stored, _ := f.EnsureTerminal(ctx, env, runner.Terminal{}, "PAY-101:coding", runner.Command{})
+	f.dead[env.ID+"/PAY-101:coding"] = true
+
+	replacement, err := f.EnsureTerminal(ctx, env, stored, "PAY-101:coding", runner.Command{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if replacement.ID == stored.ID {
+		t.Fatalf("replacement ID = stored ID %q", stored.ID)
+	}
+	if f.createN != 2 {
+		t.Fatalf("CreateTerminal calls = %d, want 2", f.createN)
+	}
+}
+
+func TestCreateTerminalAlwaysCreates(t *testing.T) {
+	f := newFakeRunner()
+	ctx := context.Background()
+	env, _ := f.EnsureEnvironment(ctx, spec())
+
+	first, _ := f.CreateTerminal(ctx, env, "PAY-101:coding", runner.Command{})
+	second, _ := f.CreateTerminal(ctx, env, "PAY-101:coding", runner.Command{})
+	if first.ID == second.ID {
+		t.Fatalf("CreateTerminal reused terminal %q", first.ID)
+	}
 }