freesolo-flash-dev 1.2.123__tar.gz → 1.2.124__tar.gz

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.123
+Version: 1.2.124
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/cli/commands/env/testing/eval.py

@@ -202,27 +202,6 @@ def _score_case(
         )
 
 
-_READY_DEPLOYMENT_STATES = frozenset({"ready", "deployed"})
-_BUSY_DEPLOYMENT_STATES = frozenset({"queued", "smoke_testing", "reconciling"})
-
-
-def _live_deployment(client, run_id: str) -> dict | None:
-    """The run's deployment record, whatever state it is in, or None when it has none.
-
-    Match by run id because `deployment_for` misses step deployments. Busy records name the rollout
-    target and hide the revision that may still be serving.
-    """
-    for entry in client.deployments() or ():
-        listed = entry.get("deployment") or {}
-        if run_id not in (listed.get("run_id"), entry.get("run_id")):
-            continue
-        # the listing omits undeployed/dry_run rows, so anything here is servable.
-        if not listed.get("run_id") and entry.get("run_id"):
-            listed = {**listed, "run_id": entry["run_id"]}
-        return listed
-    return None
-
-
 def _case_ids(cases: list[EvalCase]) -> list[str]:
     """Stable per-case ids, disambiguating sidecars that reuse one id across cases.
 

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/cli/scaffold/TRAINING.md

@@ -681,7 +681,7 @@ spending another GPU run:
 | Local-only env path in config                                | Config validation says there is no local path mode                                                                                                                                       | Publish first, then use the returned slug in `[environment] id`. `flash train` only runs published env ids, not local paths.                                                                                                                                                                                                                                                                                                    |
 | Config knobs are in the wrong table                          | Validation rejects `[grpo]`, `[sft]`, or unknown `[train]` keys                                                                                                                          | Put `epochs`, `group_size`, `max_completion_tokens`, `temperature`, `max_context_tokens`, LoRA, and other training knobs under `[train]`.                                                                                                                                                                                                                                                                                       |
 | GPU selection is not what you expected                       | Leaving `[gpu] type` unset may select a different fitting class as prices or capacity change                                                                                             | Set `[gpu] type` to an active validated class to hard-pin it, to a list (`["A100 PCIe", "A100 SXM"]`) to allow several, or leave it unset for managed cheapest-fit allocation. `train.hf_repo` remains platform-managed.                                                                                                                                                                                                        |
-| A pinned run stops after two capacity refusals               | The pinned class had no capacity, and no other class was allowed, so there was nowhere to fail over                                                                                      | Widen the search: drop the `[gpu] type` pin for managed allocation, name alternatives with `type = ["A100 PCIe", "A100 SXM"]`, or drop `[gpu] provider`. Flash stops once the class a retry would land on has refused capacity twice, rather than re-queueing on a class the provider keeps refusing. One refusal always earns a retry, since a momentary shortage is not a sold-out class.                                     |
+| A run stops after an infrastructure or OOM failure           | No candidate with strictly more usable VRAM remains, even if another provider offers the same-size GPU                                                                                   | Widen `[gpu] type` or the card-count ceiling to include a strictly larger fitting shape, or reduce the workload footprint. Flash preserves allocator order among larger candidates and never retries an equal-size shape. A managed shared-cache `no_capacity` or `poll_error` gets one cacheless same-shape retry before this rule applies.                                                                                    |
 | Secrets are not available on the worker                      | Reward code works locally but remote logs show missing API keys or auth failures                                                                                                         | List secret names under `[environment] secrets = [...]`, export those env vars locally before submit, or put them in local `.env` / `.env.local`. Never hard-code secret values in the config.                                                                                                                                                                                                                                  |
 | Scorer dependency is missing on the worker                   | Reward code works locally but every reward is `0.0` remotely; the import your scorer needs is installed in your venv, not the worker's                                                   | List the packages your scorer imports under `[environment] pip = ["pymongo>=4.6"]`. They are installed alongside Flash's own worker requirement. `flash env test` names the failing import locally before you spend a GPU on it. Entries name packages only: pip options and URLs with inline credentials are rejected, because the spec is stored and uploaded in plaintext (put the credential in `[environment] secrets`).   |
 | Wrong model / thinking setting                               | Config validation fails, or chat behavior does not match the run                                                                                                                         | Config validation is authoritative for model and thinking compatibility. Thinking is a run-level choice, and `flash models chat` does not expose an override flag.                                                                                                                                                                                                                                                              |
@@ -690,7 +690,7 @@ spending another GPU run:
 | Reward rises but behavior is worse                           | Short, templated, malformed, or reward-hacked outputs score well                                                                                                                         | Deploy the adapter and probe real examples. Add hard validity gates before judge calls, penalize degenerate shortcuts, and judge the outcome rather than the surface string.                                                                                                                                                                                                                                                    |
 | OPD makes the student worse, not better                      | The distilled adapter scores _below_ its SFT/base start even though the per-token loss fell                                                                                              | The teacher, not a knob, is the ceiling. Reverse-KL only pulls the student toward the managed GLM-5.2 teacher, so a teacher that is weak or wrong on _your_ task transfers its mistakes. Vet the teacher before you spend — the "On-policy distillation" section gives the two ways to do that with a managed key. If it can't beat your student, use GRPO or SFT instead — OPD cannot exceed a teacher that can't do the task. |
 | Output is truncated                                          | Correct-looking answers cut off mid-response or JSON is incomplete                                                                                                                       | Increase `max_completion_tokens` for GRPO/OPD rollouts or `max_context_tokens` for total prompt+completion context only after seeing truncation. Oversizing them by default just burns memory/cost.                                                                                                                                                                                                                             |
-| Infrastructure, CUDA, OOM, vLLM, or kernel failure           | Run errors before useful metrics, often during setup/model load                                                                                                                          | Treat this as infrastructure pressure, not proof the model is too large. Read `flash runs log <run-id>`, reduce footprint (`max_context_tokens`, `max_completion_tokens`, `group_size`) if needed, and let Flash retry/allocate another fitting GPU class.                                                                                                                                                                      |
+| Infrastructure, CUDA, OOM, vLLM, or kernel failure           | Run errors before useful metrics, often during setup/model load                                                                                                                          | Read `flash runs log <run-id>`. Flash advances a single executed-width-aware usable-VRAM floor for candidate-bound infrastructure and OOM failures, then retries only a strictly larger fitting candidate while keeping infrastructure and OOM budgets separate. Reduce `max_context_tokens`, `max_completion_tokens`, or `group_size` when no larger candidate remains.                                                        |
 | Run looks stuck after disconnecting                          | Terminal stopped streaming but the job may still be alive                                                                                                                                | Ctrl-C detaches. Use `flash runs log <run-id> --follow` to reattach, `flash runs log <run-id>` for the console/error output, or `flash runs cancel <run-id>` if you intentionally want to stop it.                                                                                                                                                                                                                              |
 | Two runs training the same thing                             | `flash runs list` shows a live run you thought you had killed, on top of the one you just submitted                                                                                      | Killing the `flash train` client only detaches; it never cancels. Always `flash runs list` and cancel the stale run _before_ re-submitting, and never wrap `flash train` in `timeout`.                                                                                                                                                                                                                                          |
 | Wrong identity spends the money                              | Push/train lands in an org you did not expect, or a run id you know is valid comes back "unknown"                                                                                        | `FREESOLO_API_KEY` in the environment silently overrides `flash login`. Run `flash whoami` first. Run visibility is scoped to the key that created the run, so archive result baselines to disk rather than relying on `flash runs list` to find them later.                                                                                                                                                                    |

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/engine/worker/train/entry/opd_train_runner.py

@@ -318,6 +318,7 @@ def _write_child_shims(
         ("train/opd/child/plugin.py", "flash_opd_plugin.py"),
         ("train/opd/child/bridge.py", "flash_opd_bridge.py"),
         ("train/opd/child/structured.py", "flash_opd_structured.py"),
+        ("train/opd/child/tensors.py", "flash_opd_tensors.py"),
         ("train/opd/child/multiturn.py", "flash_opd_multiturn.py"),
         ("train/opd/child/entry.py", "flash_opd_entry.py"),
         ("train/opd/child/replay_guard.py", "flash_opd_replay_guard.py"),

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/engine/worker/train/entry/sft_train.py

@@ -112,26 +112,6 @@ def _warmstart_adapter_path(
     return adapter_dir
 
 
-def _verl_image_message_content(content) -> str:
-    if isinstance(content, str):
-        return content
-    if content is None:
-        return ""
-    if not isinstance(content, list):
-        raise ValueError("multimodal message content must be text or content blocks")
-    parts: list[str] = []
-    for block in content:
-        if not isinstance(block, dict):
-            raise ValueError("multimodal message content blocks must be objects")
-        if block.get("type") == "text":
-            parts.append(str(block.get("text") or ""))
-        elif block.get("type") == "image":
-            parts.append("<image>")
-        else:
-            raise ValueError(f"unsupported multimodal SFT content block {block.get('type')!r}")
-    return "".join(parts)
-
-
 _restore_verl_resume = partial(restore_verl_resume, job_label="SFT")
 
 

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/engine/worker/train/opd/child/plugin.py

@@ -26,6 +26,15 @@ if PLUGIN_LOADED_EXTERNALLY:
     from flash_multiturn_glue import multi_modal_image_count
     from flash_opd_multiturn import build_flash_multi_turn_agent_loop
     from flash_opd_structured import StructuredOutputReplay, canonical_structured_spec
+    from flash_opd_tensors import (
+        full_sequence_signal_sequences as _full_sequence_signal_sequences,
+    )
+    from flash_opd_tensors import (
+        packed_full_sequence as _packed_full_sequence,
+    )
+    from flash_opd_tensors import (
+        signal_sequences as _signal_sequences,
+    )
 else:
     from flash.engine.worker.train.core.child.glue import multi_modal_image_count
     from flash.engine.worker.train.opd.child.multiturn import build_flash_multi_turn_agent_loop
@@ -33,6 +42,15 @@ else:
         StructuredOutputReplay,
         canonical_structured_spec,
     )
+    from flash.engine.worker.train.opd.child.tensors import (
+        full_sequence_signal_sequences as _full_sequence_signal_sequences,
+    )
+    from flash.engine.worker.train.opd.child.tensors import (
+        packed_full_sequence as _packed_full_sequence,
+    )
+    from flash.engine.worker.train.opd.child.tensors import (
+        signal_sequences as _signal_sequences,
+    )
 
 _PERMANENT_TEACHER_EXIT = 86
 _TRANSIENT_TEACHER_EXIT = 87
@@ -86,16 +104,6 @@ def deterministic_rollout_seed(
     return int.from_bytes(digest, "big") & ((1 << 63) - 1)
 
 
-def _signal_sequences(group_ids, response_mask):
-    """Return the per-sequence mask for rows carrying at least one aligned student token."""
-    return ((group_ids >= 0) & response_mask.bool()).any(dim=-1)
-
-
-def _full_sequence_signal_sequences(group_ids):
-    """Detect aligned metadata in native full-sequence teacher tensors."""
-    return group_ids.ge(0).flatten(start_dim=1).any(dim=-1)
-
-
 def _flash_groupwise_reverse_kl_values(
     student_logprobs,
     teacher_logsums,
@@ -408,8 +416,14 @@ def _register_flash_distillation_loss() -> None:
     def flash_groupwise_reverse_kl(config, distillation_config, model_output, data):
         _set_current_global_batch_info(config, data)
         student_logprobs = no_padding_2_padding(model_output["log_probs"], data)
-        teacher_logsums = no_padding_2_padding(data["teacher_logprobs"], data).squeeze(-1)
-        group_ids = no_padding_2_padding(data["teacher_ids"], data).squeeze(-1).long()
+        teacher_logsums = no_padding_2_padding(
+            _packed_full_sequence(data["teacher_logprobs"], data), data
+        ).squeeze(-1)
+        group_ids = (
+            no_padding_2_padding(_packed_full_sequence(data["teacher_ids"], data), data)
+            .squeeze(-1)
+            .long()
+        )
         response_mask = data["response_mask"]
         if response_mask.is_nested:
             response_mask = response_mask.to_padded_tensor(False)

freesolo_flash_dev-1.2.124/flash/engine/worker/train/opd/child/tensors.py

@@ -0,0 +1,56 @@
+"""Batch-layout helpers for the flash OPD distillation loss.
+
+verl hands the loss a `TensorDict` whose fields are laid out per field rather than per batch:
+`list_of_dict_to_tensordict` stacks a field when every sample's tensor shares one shape and
+nests it only when the shapes differ. So `prompts`, `teacher_logprobs`, and `teacher_ids` can
+each arrive nested or padded independently, and the loss has to read all of them either way.
+"""
+
+from __future__ import annotations
+
+
+def signal_sequences(group_ids, response_mask):
+    """Return the per-sequence mask for rows carrying at least one aligned student token."""
+    return ((group_ids >= 0) & response_mask.bool()).any(dim=-1)
+
+
+def full_sequence_signal_sequences(group_ids):
+    """Detect aligned metadata in native full-sequence teacher tensors.
+
+    Padded only: `flatten` raises on a nested tensor's ragged dim, so callers holding a nested
+    tensor must reduce per row instead.
+    """
+    return group_ids.ge(0).flatten(start_dim=1).any(dim=-1)
+
+
+def packed_full_sequence(tensor, data):
+    """Put a full-sequence teacher tensor into the layout `no_padding_2_padding` documents.
+
+    That helper wants a nested tensor or `(total_nnz, *)` and reads `.values()` only when
+    nested, so a padded teacher measures as `bsz` and trips its `total_nnz` assertion.
+
+    The mask decides which positions are real, never the row's shape. verl pads a teacher row on
+    both sides (`_pad_teacher_outputs` left-pads by `prompt_width - prompt_length` and right-pads
+    the response), so a length alone cannot say where the real tokens sit and selecting a prefix
+    would silently train against the pad.
+    """
+    if tensor.is_nested:
+        return tensor
+    if "attention_mask" in data:
+        return tensor[data["attention_mask"].bool()]
+    # no mask: only a nested-prompt batch may omit that key, and only such a batch carries the
+    # lengths needed to check the stack. verl stacks this field only when every sequence shares
+    # one length, and pads a row only to the strided widths, so an unmasked stack is full.
+    prompts = data["prompts"]
+    if not prompts.is_nested:
+        raise AssertionError(
+            "flash OPD strided batch carries no attention mask, so real token positions "
+            "cannot be located"
+        )
+    lengths = prompts.offsets().diff() + data["responses"].offsets().diff()
+    if int(lengths.min()) != tensor.shape[1] or int(lengths.max()) != tensor.shape[1]:
+        raise AssertionError(
+            "flash OPD teacher tensor is padded but the batch carries no attention mask: "
+            f"width {tensor.shape[1]} against lengths {int(lengths.min())}..{int(lengths.max())}"
+        )
+    return tensor.flatten(0, 1)

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/core/base.py

@@ -929,9 +929,8 @@ class Provider(Protocol):
 
     # NOTE: ``supports_weight_cache: bool`` is an OPTIONAL capability attr (read via
     # ``getattr(prov, "supports_weight_cache", False)``, off this Protocol for the same isinstance
-    # reason as below): True only for the provider that offers the shared weight-cache network volume
-    # (RunPod). The runner gates its one-shot cache-less retry fallback on it; every other provider
-    # defaults False.
+    # reason as below): True for providers that attach the managed shared weight cache. The runner
+    # gates its one-shot cache-less retry fallback on it; every other provider defaults False.
 
     # ``run_instances_remaining(run_id)`` is optional and must stay off this runtime-checkable
     # protocol. callers use getattr in server/_runtime.py. ``[]`` confirms clear; non-empty means a

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/lambda_/execution/provider.py

@@ -23,6 +23,7 @@ class LambdaProvider(InstanceProvider):
     """``base.Provider`` for the Lambda Cloud substrate."""
 
     name = "lambda"
+    supports_weight_cache = True
     _gpu_identity_attr = "lambda_name"
 
     @property

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/lambda_/jobs/__init__.py

@@ -314,7 +314,7 @@ def _rent_instance(
     """Rent one box and return its handle, owing nothing rented behind on any exit.
 
     The whole rent-to-handle window lives here because its ORDER is the correctness argument, and
-    both call sites (the region walk and its cache-less retry) need exactly the same order:
+    every region in the launch walk needs exactly the same order:
 
     - ``arm`` immediately before the request and only then. A deadline miss in the precheck rents
       nothing, and an armed guard there would reap by run label, killing every other concurrent
@@ -356,46 +356,6 @@ def _rent_instance(
         raise
 
 
-def _retry_launch_without_cache(
-    plan: _LaunchPlan, inst: LambdaInstance, say, reap
-) -> tuple[LambdaJobHandle | None, Exception | None]:
-    """Rent a cache-less box for this region; return ``(handle, None)`` or ``(None, rejection)``.
-
-    The arm/own/return ordering lives in ``_rent_instance``. What is specific here is the
-    rejection policy, which mirrors the main walk's: a CLEAN reject rented nothing and stands the
-    guard down before anything below can raise, while an AMBIGUOUS one may have billed a box and
-    keeps the guard armed across reconciliation.
-    """
-    say(f"retrying {inst.region} WITHOUT the weight cache (attach may have caused the reject)")
-    try:
-        return _rent_instance(
-            plan,
-            inst,
-            say,
-            reap,
-            user_data=plan.cold_user_data,
-            file_system_names=None,
-            describe=lambda instance_id: (
-                f"launched lambda instance {instance_id} (cold, cache-less): {inst.gpu} "
-                f"{inst.instance_type} in {inst.region} attempt={plan.attempt} seed={plan.seed}"
-            ),
-        ), None
-    except lambda_api.LambdaApiError as error:
-        clean = _launch_rejection_is_clean(error)
-        if clean:
-            # rented nothing: stand down on the FIRST statement, before the diagnostic and the say
-            # below, either of which can raise while armed and would then reap by run label,
-            # killing every other concurrent seed over a request that rented nothing.
-            reap.disarm()
-        cold_detail = sanitize_diagnostic(error, limit=1000)
-        if not clean:
-            # may have billed a box: the guard stays ARMED across reconciliation, which always
-            # raises, so an unnamed instance still has something able to find it.
-            _abort_ambiguous_launch(plan.spec.run_id, type(error).__name__)
-        say(f"region {inst.region} also rejected cold: {cold_detail}")
-        return None, error
-
-
 def _refresh_launch_candidates(
     inst: LambdaInstance,
     tried_regions: set[str],
@@ -495,11 +455,7 @@ def _build_launch_plan(
 def _region_launch_inputs(
     plan: _LaunchPlan, inst: LambdaInstance, say
 ) -> tuple[str | None, list[str] | None, Exception | None]:
-    """Resolve one region's ``(user_data, file_system_names)``, falling back cold on a cache miss.
-
-    Returns ``(None, None, error)`` when the region must be skipped: a preload that trained on a
-    cache-less box would warm nothing, so it walks on instead of falling back.
-    """
+    """Resolve one region's cached launch inputs or return its attachment failure."""
     if not plan.cache_name:
         return plan.cold_user_data, None, None
     try:
@@ -527,14 +483,9 @@ def _region_launch_inputs(
         return region_user_data, [plan.cache_name], None
     except Exception as e:
         detail = sanitize_diagnostic(e, limit=1000)
-        # preload must not fall back cold because it would train instead of warming the cache.
-        if plan.mode == "preload":
-            say(
-                f"weight cache unavailable in {inst.region} ({detail}); skipping (preload needs it)"
-            )
-            return None, None, e
-        say(f"weight cache unavailable in {inst.region} ({detail}); launching cold")
-        return plan.cold_user_data, None, e
+        suffix = " (preload needs it)" if plan.mode == "preload" else ""
+        say(f"weight cache unavailable in {inst.region} ({detail}); skipping{suffix}")
+        return None, None, e
 
 
 def launch_and_submit(
@@ -620,24 +571,6 @@ def launch_and_submit(
                     )
                     _abort_ambiguous_launch(spec.run_id, type(e).__name__)
                 say(f"region {inst.region} ({inst.gpu} {inst.instance_type}) rejected: {detail}")
-                # Filesystem-attach errors: retry once without the cache before walking (clean reject = safe).
-                fs_attach_reject = fs_names and any(
-                    tok in str(e).lower() for tok in ("file_system", "filesystem", "file-system")
-                )
-                if mode != "preload" and fs_attach_reject:
-                    # the cache-less retry rents its own box, and arms the guard itself immediately
-                    # before that request: an interrupt while it is in flight can leave an instance
-                    # created but unnamed, and only the run-label reap can find it. Arming across
-                    # the whole call instead would reap on paths that rented nothing (a deadline
-                    # miss in the precheck, a raising preamble) or that already cleaned up exactly.
-                    handle, last_err = _retry_launch_without_cache(plan, inst, say, reap)
-                    if handle is not None:
-                        # the retry's box is rented and owned by exact id; keep the guard holding
-                        # it until the caller has the handle, exactly as the primary path does.
-                        return handle
-                    # nothing stayed rented: the retry disarmed on its reject paths, and this
-                    # keeps the walk's next region from inheriting a guard it did not arm.
-                    reap.disarm()
                 # Preload must not refresh to a different region (would warm the wrong one).
                 if mode != "preload" and not candidates and not refreshed:
                     refreshed = True

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/lambda_/jobs/reap.py

@@ -21,8 +21,8 @@ class _CoarseReapGuard:
 
     Armed for exactly the window a launch request is in flight with no instance id in hand, when
     the label reap is the ONLY thing that can find a box that is rented but not yet named. It is
-    an object rather than a local so the cache-less retry can arm it around its own request
-    instead of the caller arming across that whole call.
+    an object rather than a local so each region can arm it around its own request instead of the
+    caller arming across the whole walk.
 
     ``owns`` narrows the guard the instant an id exists. Ownership transfer would otherwise span
     two statements (the publication helper returns, then the caller disarms), and an interrupt

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/runpod/execution/job_execution.py

@@ -189,7 +189,6 @@ def submit_run(
         failure_detail_reader=failure_reader,
         current_attempt=attempt_id,
         **deadline_kwargs(runpod_polling.poll_job, deadline_at),
-        on_last_gpu=on_last_gpu,
         # the count actually rented for this attempt, which allocation may have resolved to fewer
         # cards than the spec's ceiling named -- so read the effective spec, not the run's request.
         **stall_kwargs(on_last_gpu=on_last_gpu, gpu_count=gpu_count_of(spec)),

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/runpod/execution/jobs.py

@@ -25,7 +25,6 @@ FAILURE_TEXT_LIMIT = 64_000
 __all__ = [
     "JobHandle",
     "PollResult",
-    "capacity_escalation_note",
     "capacity_grace_multiplier",
     "decode_output",
 ]
@@ -33,16 +32,12 @@ __all__ = [
 
 # capacity grace on the LAST candidate class: there is nowhere left to walk, so wait longer before
 # giving up. purely a timing knob -- it says nothing about whether a retry follows, because
-# on_last_gpu (runner/lifecycle.py) is also true when the infra retry budget is exhausted.
+# on_last_gpu (runner/supervise/retry_decision.py) is also true when the infra retry budget is exhausted.
 LAST_GPU_CAPACITY_GRACE_S = 900.0
 
 # multi-card shapes are rarer than single cards, so a grace sized for 1x expires on a 4x wait that
-# was merely slow rather than starved. expiring it does not find capacity faster: the supervisor
-# tears the endpoint down and re-requests THE SAME class (`_select_candidate` re-picks the only
-# fitting shape once nothing is untried), so the run pays a fresh cold start to rejoin the same
-# queue it just left. observed: 3-5 attempts and ~55 min of queueing per arm before a single
-# optimizer step, with the multi-GPU arms churning most. scale the wait with the card count
-# instead, so scarcity is waited out on one queue position rather than re-requested.
+# was merely slow rather than starved. scale the wait with the card count so scarcity is waited out
+# on one queue position rather than paying another cold start.
 CAPACITY_GRACE_PER_GPU_CAP = 4
 
 
@@ -209,19 +204,6 @@ class GraceTimer:
         self.seen = now
 
 
-def capacity_escalation_note(on_last_gpu: bool) -> str:
-    """The escalation clause a capacity failure detail carries. States the escalation fact ONLY.
-
-    `on_last_gpu` means no further class escalation follows, not that classes are exhausted or a
-    retry occurs. Keep both branches neutral; the supervisor owns target selection and budget.
-    """
-    return (
-        "no further GPU-class escalation follows"
-        if on_last_gpu
-        else "GPU-class escalation may follow"
-    )
-
-
 # heartbeat readers and provenance checks are provider-neutral and live in
 # flash.providers.artifacts.hf; this module imports only the predicate it executes.
 

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/runpod/execution/polling.py

@@ -30,7 +30,6 @@ from flash.providers.runpod.execution.jobs import (
     GraceTimer,
     _append_failure_artifacts,
     _safe_failure_text,
-    capacity_escalation_note,
     decode_output,
     surfaced_worker_flags,
 )
@@ -61,7 +60,6 @@ class _PollContext:
     absolute_deadline: float | None
     current_attempt: int
     launch_ts: float
-    next_gpu_note: str
     poll_errors: PollErrorTracker
 
 
@@ -233,7 +231,7 @@ def _classify_queue_state(
             failure="no_capacity",
             detail=f"never scheduled: job stuck IN_QUEUE for "
             f"{int(now - state.queued_timer.since)}s "
-            f"(no RunPod capacity for the pinned GPU class); {context.next_gpu_note}",
+            "(no RunPod capacity for the pinned GPU class)",
         )
     if status != "IN_QUEUE":
         # the in-queue grace timers measure continuous throttle/unhealthy while queued (like
@@ -307,8 +305,8 @@ def _classify_queue_state(
                 False,
                 failure="no_capacity",
                 detail=f"never scheduled: worker stuck THROTTLED for "
-                f"{int(now - state.throttled_timer.since)}s while IN_QUEUE (no RunPod "
-                f"capacity for the pinned GPU class); {context.next_gpu_note}",
+                f"{int(now - state.throttled_timer.since)}s while IN_QUEUE "
+                "(no RunPod capacity for the pinned GPU class)",
             )
     except Exception:
         pass
@@ -456,12 +454,11 @@ def poll_job(
     queue_grace_s: float = 300.0,
     deadline_at: float | None = None,
     current_attempt: int | None = None,
-    on_last_gpu: bool = False,
 ) -> PollResult:
     """Poll a queue job to completion; resilient to transient API errors.
 
     Use setup grace before the first heartbeat, then stall grace; fail fast on sustained throttled,
-    unhealthy, or over-queued states. `on_last_gpu` states only whether escalation remains.
+    unhealthy, or over-queued states.
     """
     if not handle.job_id:
         raise ValueError("endpoint-only RunPod handles cannot be polled")
@@ -487,7 +484,6 @@ def poll_job(
         absolute_deadline=absolute_deadline,
         current_attempt=attempt_id,
         launch_ts=launch_ts,
-        next_gpu_note=capacity_escalation_note(on_last_gpu),
         poll_errors=PollErrorTracker(say, interval_s),
     )
     state = _PollState(

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/providers/runpod/execution/provider.py

@@ -155,10 +155,6 @@ class RunpodProvider:
             failure_detail_reader=failure_reader,
             current_attempt=rh.attempt,
             **deadline_kwargs(runpod_polling.poll_job, _deadline_at),
-            # the persisted scarcity flag controls stall grace, not capacity wording. recovery
-            # rebuilds the unpinned allocation with a fresh candidate set, so claiming no escalation
-            # remains would be false.
-            #
             # the card count comes from the spec rather than the handle's `allocated_gpu_count`:
             # attach polls the persisted EFFECTIVE worker spec, which submission already stamped
             # with the count allocation resolved, and the attach context pops that handle key off

{freesolo_flash_dev-1.2.123 → freesolo_flash_dev-1.2.124}/flash/runner/accounting/reconciliation.py

@@ -99,31 +99,6 @@ def _compare_and_clear_remote(run_id: str, expected_remote: dict) -> bool:
     return True
 
 
-def _compare_and_prepare_resubmit(
-    run_id: str,
-    expected_remote: dict | None,
-    *,
-    expected_state: str | None = None,
-) -> bool:
-    """Claim a nonterminal recovery launch only while its expected remote still owns the run."""
-    report_status: RunStatus | None = None
-    with state._status_guard(run_id):
-        status = status_ops.get_status(run_id)
-        if status.state in state.TERMINAL_STATES:
-            return False
-        if expected_state is not None and status.state != expected_state:
-            return False
-        if not _expected_remote_matches(status.remote, expected_remote):
-            return False
-        status.state = "provisioning"
-        status.updated_at = time.time()
-        state._save_status_unlocked(status)
-        report_status = status
-    if report_status is not None:
-        reporting._report_status(report_status)
-    return True
-
-
 def _compare_and_fail_remote(
     run_id: str,
     expected_remote: dict | None,
@@ -180,7 +155,7 @@ def _compare_and_complete_remote(
     expected_attempt = (
         expected_remote.get("attempt")
         if isinstance(expected_remote, dict)
-        else attempts._latest_reserved_attempt(run_id)
+        else attempts.latest_reserved_attempt(run_id)
     )
     metrics, verified_attempt = status_ops.validate_terminal_source_metrics(
         status,
@@ -236,15 +211,18 @@ def _canonical_cleanup_remote(remote: object) -> dict | None:
         if provider == "runpod":
             from flash.providers.runpod.execution.jobs import JobHandle as RunpodJobHandle
 
-            return RunpodJobHandle.from_dict(remote).to_dict()
-        if provider == "lambda":
+            record = RunpodJobHandle.from_dict(remote).to_dict()
+        elif provider == "lambda":
             from flash.providers.lambda_.jobs.builders import LambdaJobHandle
 
-            return LambdaJobHandle.from_dict(remote).to_dict()
-        if provider == "vast":
+            record = LambdaJobHandle.from_dict(remote).to_dict()
+        elif provider == "vast":
             from flash.providers.vast.jobs.builders import VastJobHandle
 
-            return VastJobHandle.from_dict(remote).to_dict()
+            record = VastJobHandle.from_dict(remote).to_dict()
+        else:
+            return None
+        return record
     except (TypeError, ValueError):
         return None
     return None