synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

synth_ai/api/train/cli.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import asyncio
 import importlib
 import os
+import time
 from collections.abc import Callable, Mapping
 from pathlib import Path
 from typing import Any, cast
@@ -16,10 +18,18 @@ try:
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load backend configuration helpers") from exc
 
+from synth_ai.streaming import (
+    CLIHandler,
+    JobStreamer,
+    LossCurveHandler,
+    StreamConfig,
+    StreamEndpoints,
+    StreamType,
+)
+
 from .builders import build_rl_payload, build_sft_payload
 from .config_finder import discover_configs, prompt_for_config
 from .env_resolver import KeySpec, resolve_env
-from .pollers import RLJobPoller, SFTJobPoller
 from .task_app import check_task_app_health
 from .utils import (
     REPO_ROOT,
@@ -36,20 +46,41 @@ from .utils import (
 )
 
 
-def _discover_dataset_candidates(config_path: Path, limit: int = 50) -> list[Path]:
+def _discover_dataset_candidates(
+    config_path: Path, limit: int = 50, timeout: float = 10.0
+) -> list[Path]:
+    root = config_path.parent
+    parent = root.parent
+    cwd = Path.cwd()
+
     search_dirs: list[Path] = [
-        config_path.parent,
-        config_path.parent / "datasets",
-        REPO_ROOT / "traces",
+        root,
+        root / "datasets",
+        parent,
+        parent / "datasets",
+        parent / "ft_data",
+        cwd,
+        cwd / "datasets",
+        cwd / "ft_data",
         REPO_ROOT / "datasets",
+        REPO_ROOT / "ft_data",
+        REPO_ROOT / "traces",
     ]
 
     candidates: list[Path] = []
     seen: set[Path] = set()
+    start = time.monotonic()
+    timed_out = False
     for directory in search_dirs:
+        if timed_out or time.monotonic() - start > timeout:
+            timed_out = True
+            break
         if not directory.exists() or not directory.is_dir():
             continue
         for path in directory.rglob("*.jsonl"):
+            if time.monotonic() - start > timeout:
+                timed_out = True
+                break
             try:
                 resolved = path.resolve()
             except OSError:
@@ -113,6 +144,62 @@ def _default_backend() -> str:
     return f"{base}/api" if not base.endswith("/api") else base
 
 
+_DEFAULT_SFT_HIDDEN_EVENTS = {
+    "sft.created",
+    "sft.pricing.check.requested",
+    "sft.pricing.check.allowed",
+    "sft.stage",
+    "snapshot.fetch",
+    "hatchet.preflight",
+    "hatchet.submission.attempt",
+    "hatchet.submission.result",
+    "sft.running",
+    "sft.status",
+    "sft.worker.alive",
+    "sft.dispatch.selected",
+    "sft.config.prepared",
+    "sft.strategy.selected",
+    "sft.training.args",
+}
+
+_DEFAULT_RL_HIDDEN_SUBSTRINGS = {"modal", "hatchet"}
+
+
+def _build_stream_components(
+    stream_format: str,
+    *,
+    hidden_event_types: set[str] | None = None,
+    hidden_event_substrings: set[str] | None = None,
+) -> tuple[StreamConfig, list]:
+    """Return stream configuration and handlers for the requested format."""
+    if stream_format == "chart":
+        config = StreamConfig(
+            enabled_streams={StreamType.STATUS, StreamType.EVENTS, StreamType.METRICS},
+            event_types={
+                "sft.progress",
+                "sft.training.started",
+                "sft.training.finish",
+                "sft.validation.summary",
+                "rl.train.step",
+                "rl.train.started",
+                "rl.train.completed",
+                "workflow.completed",
+                "workflow.failed",
+            },
+            metric_names={"train.loss"},
+        )
+        handlers = [LossCurveHandler()]
+    else:
+        config = StreamConfig.default()
+        handlers = [
+            CLIHandler(
+                hidden_event_types=hidden_event_types or set(),
+                hidden_event_substrings=hidden_event_substrings or set(),
+            )
+        ]
+    return config, handlers
+
+
 @click.command("train")
 @click.option(
     "--config",
@@ -161,6 +248,13 @@ def _default_backend() -> str:
     "--poll-timeout", default=3600.0, type=float, help="Maximum seconds to poll before timing out"
 )
 @click.option("--poll-interval", default=5.0, type=float, help="Seconds between poll attempts")
+@click.option(
+    "--stream-format",
+    type=click.Choice(["cli", "chart"]),
+    default="cli",
+    show_default=True,
+    help="Streaming output style (cli = line updates, chart = live loss panel)",
+)
 @click.option(
     "--examples",
     "examples_limit",
@@ -182,6 +276,7 @@ def train_command(
     poll: bool,
     poll_timeout: float,
     poll_interval: float,
+    stream_format: str,
     examples_limit: int | None,
 ) -> None:
     """Interactive launcher for RL / SFT jobs."""
@@ -280,6 +375,7 @@ def train_command(
             poll=poll,
             poll_timeout=poll_timeout,
             poll_interval=poll_interval,
+            stream_format=stream_format,
         )
     else:
         dataset_override_path = Path(dataset_path).expanduser().resolve() if dataset_path else None
@@ -293,14 +389,23 @@ def train_command(
             poll=poll,
             poll_timeout=poll_timeout,
             poll_interval=poll_interval,
+            stream_format=stream_format,
             examples_limit=examples_limit,
         )
 
 
 def _wait_for_training_file(
-    backend_base: str, api_key: str, file_id: str, *, timeout: float = 120.0
+    backend_base: str, api_key: str, file_id: str, *, timeout: float = 10.0
 ) -> None:
-    url = f"{backend_base}/learning/files/{file_id}"
+    """Wait for training file to be visible after upload.
+    
+    Reduced from 120s to 10s because:
+    - POST response already confirms file is uploaded
+    - Backend now forces read-your-writes consistency  
+    - By job creation time, replica lag has resolved
+    - Quick sanity check only, not critical path
+    """
+    url = f"{backend_base.rstrip('/')}/files/{file_id}"
     headers = {"Authorization": f"Bearer {api_key}"}
     elapsed = 0.0
     interval = 2.0
@@ -378,6 +483,7 @@ def handle_rl(
     poll: bool,
     poll_timeout: float,
     poll_interval: float,
+    stream_format: str,
 ) -> None:
     overrides: dict[str, Any] = {
         "backend": backend_base,
@@ -475,10 +581,25 @@ def handle_rl(
         click.echo(f"Created job {job_id} (polling disabled)")
         return
 
-    poller = RLJobPoller(backend_base, synth_key, interval=poll_interval, timeout=poll_timeout)
-    outcome = poller.poll_job(job_id)
-    click.echo(f"Final status: {outcome.status}")
-    click.echo(preview_json(outcome.payload, limit=600))
+    click.echo("\n=== Streaming Job Progress ===")
+    config, handlers = _build_stream_components(
+        stream_format, hidden_event_substrings=_DEFAULT_RL_HIDDEN_SUBSTRINGS
+    )
+    if stream_format == "chart":
+        click.echo("Using live loss chart (metric=train.loss)")
+    streamer = JobStreamer(
+        base_url=backend_base,
+        api_key=synth_key,
+        job_id=job_id,
+        endpoints=StreamEndpoints.rl(job_id),
+        config=config,
+        handlers=handlers,
+        interval_seconds=poll_interval,
+        timeout_seconds=poll_timeout,
+    )
+    final_status = asyncio.run(streamer.stream_until_terminal())
+    click.echo(f"Final status: {final_status.get('status', 'unknown')}")
+    click.echo(preview_json(final_status, limit=600))
 
 
 def handle_sft(
@@ -492,6 +613,7 @@ def handle_sft(
     poll: bool,
     poll_timeout: float,
     poll_interval: float,
+    stream_format: str,
     examples_limit: int | None,
 ) -> None:
     dataset_path = dataset_override
@@ -524,7 +646,7 @@ def handle_sft(
             click.echo("Validating validation dataset…")
             validate_sft_jsonl(build.validation_file)
 
-        upload_url = f"{backend_base}/learning/files"
+        upload_url = f"{backend_base.rstrip('/')}/files"
         click.echo("\n=== Uploading Training Data ===")
         click.echo(f"Dataset: {build.train_file}")
         click.echo(f"Destination: {upload_url}")
@@ -579,7 +701,8 @@ def handle_sft(
         try:
             _wait_for_training_file(backend_base, synth_key, train_file_id)
         except click.ClickException as exc:
-            raise click.ClickException(f"Training file {train_file_id} not ready: {exc}") from exc
+            click.echo(f"[WARN] File readiness check failed: {exc}")
+            click.echo("Proceeding anyway - backend will validate file during job creation...")
 
         click.echo("\n=== Creating Training Job ===")
         click.echo("Job payload preview:")
@@ -618,10 +741,25 @@ def handle_sft(
             click.echo(f"Started job {job_id} (polling disabled)")
             return
 
-        poller = SFTJobPoller(backend_base, synth_key, interval=poll_interval, timeout=poll_timeout)
-        outcome = poller.poll_job(job_id)
-        click.echo(f"Final status: {outcome.status}")
-        click.echo(preview_json(outcome.payload, limit=600))
+        click.echo("\n=== Streaming Job Progress ===")
+        config, handlers = _build_stream_components(
+            stream_format, hidden_event_types=_DEFAULT_SFT_HIDDEN_EVENTS
+        )
+        if stream_format == "chart":
+            click.echo("Using live loss chart (metric=train.loss)")
+        streamer = JobStreamer(
+            base_url=backend_base,
+            api_key=synth_key,
+            job_id=job_id,
+            endpoints=StreamEndpoints.learning(job_id),
+            config=config,
+            handlers=handlers,
+            interval_seconds=poll_interval,
+            timeout_seconds=poll_timeout,
+        )
+        final_status = asyncio.run(streamer.stream_until_terminal())
+        click.echo(f"Final status: {final_status.get('status', 'unknown')}")
+        click.echo(preview_json(final_status, limit=600))
     finally:
         if limited_path is not None:
             try:

synth_ai/api/train/config_finder.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import json
-import os
 from collections.abc import Iterable
 from dataclasses import dataclass
 from pathlib import Path
@@ -11,7 +10,9 @@ import click
 from .utils import REPO_ROOT, load_toml, preview_json
 
 _SKIP_DIRS = {".git", "__pycache__", ".venv", "node_modules", "dist", "build"}
-_STATE_FILE = os.path.expanduser("~/.synth-ai/demo.json")
+
+_STATE_DIR = Path.home() / ".synth-ai"
+_STATE_FILE = _STATE_DIR / "train_cli.json"
 
 
 @dataclass(slots=True)
@@ -23,8 +24,8 @@ class ConfigCandidate:
 def _load_last_config() -> Path | None:
     """Load the last used training config path from state file."""
     try:
-        if os.path.isfile(_STATE_FILE):
-            with open(_STATE_FILE) as fh:
+        if _STATE_FILE.is_file():
+            with _STATE_FILE.open() as fh:
                 data = json.load(fh)
                 if isinstance(data, dict):
                     last_config = data.get("LAST_CONFIG")
@@ -41,14 +42,14 @@ def _save_last_config(config_path: Path) -> None:
     """Save the last used training config path to state file."""
     try:
         data = {}
-        if os.path.isfile(_STATE_FILE):
-            with open(_STATE_FILE) as fh:
+        if _STATE_FILE.is_file():
+            with _STATE_FILE.open() as fh:
                 data = json.load(fh) or {}
         if not isinstance(data, dict):
             data = {}
         data["LAST_CONFIG"] = str(config_path.resolve())
-        os.makedirs(os.path.dirname(_STATE_FILE), exist_ok=True)
-        with open(_STATE_FILE, "w") as fh:
+        _STATE_DIR.mkdir(parents=True, exist_ok=True)
+        with _STATE_FILE.open("w") as fh:
             json.dump(data, fh)
     except Exception:
         pass
@@ -77,6 +78,7 @@ def _iter_candidate_paths() -> Iterable[Path]:
         REPO_ROOT / "configs",
         REPO_ROOT / "examples",
         REPO_ROOT / "training",
+        REPO_ROOT / "synth_ai" / "demos",
     ]
     for base in preferred:
         if not base.exists():
@@ -148,6 +150,10 @@ def discover_configs(explicit: list[str], *, requested_type: str | None) -> list
             raise click.ClickException(f"Config not found: {path}")
         data = load_toml(path)
         cfg_type = _infer_config_type(data)
+        if cfg_type == "unknown":
+            raise click.ClickException(
+                f"Config {path} is missing algorithm.type/method metadata. Add type = 'rl' or 'sft'."
+            )
         candidates.append(ConfigCandidate(path=path, train_type=cfg_type))
         seen.add(path)
 
@@ -162,10 +168,12 @@ def discover_configs(explicit: list[str], *, requested_type: str | None) -> list
         except Exception:
             continue
         cfg_type = _infer_config_type(data)
+        if cfg_type == "unknown":
+            continue
         candidates.append(ConfigCandidate(path=path, train_type=cfg_type))
 
     if requested_type and requested_type != "auto":
-        candidates = [c for c in candidates if c.train_type in {requested_type, "unknown"}]
+        candidates = [c for c in candidates if c.train_type == requested_type]
 
     # De-dupe by path and keep deterministic ordering by directory depth then name
     candidates.sort(key=lambda c: (len(c.path.parts), str(c.path)))
@@ -196,9 +204,8 @@ def prompt_for_config(
 
     click.echo("Select a training config:")
     for idx, cand in enumerate(candidates, start=1):
-        label = cand.train_type if cand.train_type != "unknown" else "?"
         last_marker = " (last used)" if last_config and cand.path.resolve() == last_config else ""
-        click.echo(f"  {idx}) [{label}] {cand.path}{last_marker}")
+        click.echo(f"  {idx}) {cand.path}{last_marker}")
     click.echo("  0) Abort")
 
     choice = click.prompt("Enter choice", type=int, default=default_idx)

synth_ai/api/train/configs/__init__.py

@@ -5,10 +5,12 @@ from .rl import (
     JudgeConfig,
     JudgeOptionsConfig,
     ModelConfig,
+    RewardsConfig,
     RLConfig,
     RLServicesConfig,
     RLTrainingConfig,
     RolloutConfig,
+    RubricConfig,
     WeightSyncConfig,
 )
 from .sft import (
@@ -20,7 +22,7 @@ from .sft import (
     TrainingConfig,
     TrainingValidationConfig,
 )
-from .shared import AlgorithmConfig, ComputeConfig
+from .shared import AlgorithmConfig, ComputeConfig, LoraConfig, PolicyConfig, TopologyConfig
 
 __all__ = [
     "AlgorithmConfig",
@@ -31,13 +33,18 @@ __all__ = [
     "JobConfig",
     "JudgeConfig",
     "JudgeOptionsConfig",
+    "LoraConfig",
     "ModelConfig",
+    "PolicyConfig",
+    "RewardsConfig",
     "RLConfig",
     "RLServicesConfig",
     "RLTrainingConfig",
     "RolloutConfig",
+    "RubricConfig",
     "SFTConfig",
     "SFTDataConfig",
+    "TopologyConfig",
     "TrainingConfig",
     "TrainingValidationConfig",
     "WeightSyncConfig",

synth_ai/api/train/configs/rl.py

@@ -7,7 +7,7 @@ from typing import Any
 from pydantic import model_validator
 
 from ..utils import load_toml
-from .shared import AlgorithmConfig, ComputeConfig, ExtraModel
+from .shared import AlgorithmConfig, ComputeConfig, ExtraModel, LoraConfig, PolicyConfig
 
 
 class RLServicesConfig(ExtraModel):
@@ -48,6 +48,16 @@ class WeightSyncConfig(ExtraModel):
     verify_every_k: int | None = None
 
 
+class RewardsConfig(ExtraModel):
+    """Rewards configuration for RL training."""
+    step_rewards_enabled: bool | None = None
+    step_rewards_mode: str | None = None
+    step_rewards_indicator_lambda: float | None = None
+    step_rewards_beta: float | None = None
+    step_rewards_strategy: str | None = None
+    event_rewards_kind: str | None = None
+
+
 class RLTrainingConfig(ExtraModel):
     num_epochs: int
     iterations_per_epoch: int
@@ -59,13 +69,17 @@ class RLTrainingConfig(ExtraModel):
     learning_rate: float
     log_interval: int | None = None
     weight_sync_interval: int | None = None
+    # DEPRECATED: flat reward fields (use rewards.* instead)
     step_rewards_enabled: bool | None = None
     step_rewards_mode: str | None = None
     step_rewards_indicator_lambda: float | None = None
     step_rewards_beta: float | None = None
     step_rewards_strategy: str | None = None
     event_rewards_kind: str | None = None
+    # NEW: nested configs
     weight_sync: WeightSyncConfig | None = None
+    lora: LoraConfig | None = None
+    rewards: RewardsConfig | None = None
 
 
 class EvaluationConfig(ExtraModel):
@@ -86,9 +100,18 @@ class JudgeOptionsConfig(ExtraModel):
     max_concurrency: int | None = None
 
 
+class RubricConfig(ExtraModel):
+    """Rubric configuration for reward blending."""
+    enabled: bool = False
+    reward_blend: dict[str, float] | None = None  # env, event, outcome weights
+
+
 class JudgeConfig(ExtraModel):
     type: str | None = None
     timeout_s: int | None = None
+    enabled: bool | None = None  # Master switch for judge/rubric
+    reward_blend: dict[str, float] | None = None  # NEW: nested reward blending (replaces rubric.weights)
+    rubric: RubricConfig | None = None  # DEPRECATED: use flat fields instead
     options: JudgeOptionsConfig | None = None
 
 
@@ -96,15 +119,16 @@ class RLConfig(ExtraModel):
     algorithm: AlgorithmConfig
     services: RLServicesConfig
     compute: ComputeConfig | None = None
-    topology: dict[str, Any] | None = None
+    topology: dict[str, Any] | None = None  # DEPRECATED: use compute.topology instead
     vllm: dict[str, Any] | None = None
-    reference: dict[str, Any] | None = None
-    model: ModelConfig
-    lora: dict[str, Any] | None = None
+    reference: dict[str, Any] | None = None  # DEPRECATED: use compute.topology.reference_placement instead
+    model: ModelConfig | None = None  # DEPRECATED: use policy instead
+    policy: PolicyConfig | None = None  # NEW: unified policy (preferred)
+    lora: dict[str, Any] | None = None  # DEPRECATED: use training.lora instead
     rollout: RolloutConfig | None = None
     evaluation: EvaluationConfig | None = None
     training: RLTrainingConfig | None = None
-    rubric: dict[str, Any] | None = None
+    rubric: dict[str, Any] | None = None  # DEPRECATED: use judge.reward_blend and judge.enabled instead
     judge: JudgeConfig | None = None
     tags: dict[str, Any] | None = None
 
@@ -113,7 +137,8 @@ class RLConfig(ExtraModel):
 
     @classmethod
     def from_mapping(cls, data: Mapping[str, Any]) -> RLConfig:
-        return cls.model_validate(dict(data))
+        """Load RL config from dict/TOML mapping."""
+        return cls.model_validate(data)
 
     @classmethod
     def from_path(cls, path: Path) -> RLConfig:

synth_ai/api/train/configs/sft.py

@@ -7,7 +7,7 @@ from typing import Any
 from pydantic import Field
 
 from ..utils import load_toml
-from .shared import AlgorithmConfig, ComputeConfig, ExtraModel
+from .shared import AlgorithmConfig, ComputeConfig, ExtraModel, LoraConfig, PolicyConfig
 
 
 class JobConfig(ExtraModel):
@@ -35,6 +35,7 @@ class TrainingConfig(ExtraModel):
     mode: str | None = None
     use_qlora: bool | None = None
     validation: TrainingValidationConfig | None = None
+    lora: LoraConfig | None = None  # NEW: nested LoRA config
 
 
 class HyperparametersParallelism(ExtraModel):
@@ -65,10 +66,12 @@ class HyperparametersConfig(ExtraModel):
 class SFTConfig(ExtraModel):
     algorithm: AlgorithmConfig | None = None
     job: JobConfig
+    policy: PolicyConfig | None = None  # NEW: unified policy section
     compute: ComputeConfig | None = None
     data: SFTDataConfig | None = None
     training: TrainingConfig | None = None
     hyperparameters: HyperparametersConfig = Field(default_factory=HyperparametersConfig)
+    lora: dict[str, Any] | None = None  # DEPRECATED: use training.lora instead
     tags: dict[str, Any] | None = None
 
     def to_dict(self) -> dict[str, Any]:
@@ -76,7 +79,8 @@ class SFTConfig(ExtraModel):
 
     @classmethod
     def from_mapping(cls, data: Mapping[str, Any]) -> SFTConfig:
-        return cls.model_validate(dict(data))
+        """Load SFT config from dict/TOML mapping."""
+        return cls.model_validate(data)
 
     @classmethod
     def from_path(cls, path: Path) -> SFTConfig:

synth_ai/api/train/configs/shared.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, model_validator
 
 
 class ExtraModel(BaseModel):
@@ -15,10 +15,67 @@ class AlgorithmConfig(ExtraModel):
     variety: str
 
 
+class TopologyConfig(ExtraModel):
+    """Compute topology configuration - how GPUs are distributed across processes."""
+    type: str | None = None  # e.g., "single_node_split"
+    gpus_for_vllm: int | None = None
+    gpus_for_training: int | None = None
+    gpus_for_ref: int | None = None
+    tensor_parallel: int | None = None
+    reference_placement: str | None = None  # NEW: e.g., "none", "shared", "dedicated"
+
+
+class LoraConfig(ExtraModel):
+    """LoRA (Low-Rank Adaptation) training configuration."""
+    r: int | None = None  # Rank
+    alpha: int | None = None
+    dropout: float | None = None
+    target_modules: list[str] | None = None
+
+
 class ComputeConfig(ExtraModel):
     gpu_type: str
     gpu_count: int
     nodes: int | None = None
+    topology: TopologyConfig | None = None  # NEW: nested topology
+
+
+class PolicyConfig(ExtraModel):
+    """Unified policy configuration for both SFT and RL.
+    
+    This is the SINGLE SOURCE OF TRUTH for:
+    - What model to use (model_name or source)
+    - How to sample from it (temperature, max_tokens, etc.)
+    - How to train it (trainer_mode, label)
+    """
+    
+    # Model specification (exactly one required)
+    model_name: str | None = None  # e.g., "Qwen/Qwen3-4B"
+    source: str | None = None       # e.g., "ft:abc123" for checkpoints
+    
+    # Sampling parameters (with sensible defaults)
+    max_tokens: int = 512
+    temperature: float = 0.7
+    top_p: float = 0.95
+    top_k: int | None = None
+    repetition_penalty: float = 1.0
+    stop_sequences: list[str] | None = None
+    
+    # Training-specific
+    trainer_mode: str  # "lora", "full", "qlora"
+    label: str         # Model identifier/name
+    
+    # Optional - for distributed inference
+    inference_url: str | None = None
+    
+    @model_validator(mode="after")
+    def _ensure_exactly_one_source(self) -> PolicyConfig:
+        """Ensure exactly one of model_name or source is set."""
+        if not (bool(self.model_name) ^ bool(self.source)):
+            raise ValueError(
+                "Must set exactly one: [policy].model_name OR [policy].source"
+            )
+        return self
 
 
-__all__ = ["ExtraModel", "AlgorithmConfig", "ComputeConfig"]
+__all__ = ["ExtraModel", "AlgorithmConfig", "ComputeConfig", "PolicyConfig", "TopologyConfig", "LoraConfig"]

synth_ai/api/train/env_resolver.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from typing import Any, cast
 
 import click
+from synth_ai.utils.env import resolve_env_var
 
 from . import task_app
 from .utils import REPO_ROOT, mask_value, read_env_file, write_env_value
@@ -232,18 +233,16 @@ def _resolve_key(resolver: EnvResolver, spec: KeySpec) -> str:
             _maybe_persist(resolver, spec, env_val)
             os.environ[spec.name] = env_val
             return env_val
-        options: list[tuple[str, Callable[[], str | None]]] = []
 
-        def _enter_manual() -> str:
-            prompt = f"Enter {spec.description}" if spec.description else f"Enter {spec.name}"
-            value = click.prompt(prompt, hide_input=spec.secret).strip()
-            if not value:
-                raise click.ClickException(f"{spec.name} cannot be empty")
-            _maybe_persist(resolver, spec, value)
-            os.environ[spec.name] = value
-            return value
+        resolve_env_var(spec.name)
+        resolved_value = os.environ.get(spec.name)
+        if resolved_value:
+            click.echo(f"Found {spec.name} via secrets helper: {mask_value(resolved_value)}")
+            _maybe_persist(resolver, spec, resolved_value)
+            os.environ[spec.name] = resolved_value
+            return resolved_value
 
-        options.append(("Enter manually", _enter_manual))
+        options: list[tuple[str, Callable[[], str | None]]] = []
 
         def _pick_env() -> str | None:
             resolver.select_new_env()
@@ -276,6 +275,10 @@ def _resolve_key(resolver: EnvResolver, spec: KeySpec) -> str:
 
 def _maybe_persist(resolver: EnvResolver, spec: KeySpec, value: str) -> None:
     # Automatically save (no prompt)
+    # Skip auto-persisting TASK_APP_URL to prevent overwriting CLI overrides
+    if spec.name == "TASK_APP_URL":
+        click.echo(f"Skipping auto-persist for {spec.name} (use CLI flags to override)")
+        return
     resolver.set_value(spec.name, value)
     click.echo(f"Saved {spec.name} to {resolver.current_path}")