freesolo-flash-dev 1.2.52__tar.gz → 1.2.58__tar.gz

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/CHANGELOG.md

@@ -39,6 +39,34 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Fixed
 
+- An ordered `[gpu] type` list is now priced over every class it declares acceptable, not just the
+  first. Allocation cost-ranks the whole set, so quoting the head alone priced a shape the run may
+  never be given — an authored `["B200", "H100"]` quoted roughly 3x the H100 the allocator would
+  really rent, and the submit-time affordability check runs on that estimate, so a run the
+  organization could afford could be refused before allocation ever got to choose. A single
+  authored class is still a hard pin, and a no-fit rejection now names every acceptable class
+  rather than whichever happened to be written first.
+
+- A provider preference that exhausts every configured provider no longer falls back to quoting
+  hardware this plane cannot rent. The final fallback dropped the restriction and ranked the
+  registered RunPod pool regardless of credentials, so a vast-only plane could quote a B200: the
+  estimate passed affordability and the run was recorded, only for live allocation to fail. The
+  quote now raises against the eligible set, which names the real constraint.
+
+- A GitHub token that is invalid, expired, or missing a scope now fails during preparation instead
+  of after a GPU is rented. Non-rate-limit 401 and 403 responses were left as untyped errors, which
+  the submit-time preflight treats as a transient blip worth deferring, so the run proceeded
+  through affordability and allocation and hit the same permanent credential error on the worker.
+  They are now classified permanent, like 404 and 422. The rate-limit 403 is still transient.
+
+- A slow heartbeat upload no longer stalls every other heartbeat behind it. The throttle clock only
+  advances once a commit lands, so during a slow HuggingFace commit other callers still saw an
+  upload as due and blocked on the upload lock for up to its acquire timeout — 30s, or 120s on a
+  critical stage — which stalled the liveness daemon and delayed leaving the wrapped stage by the
+  same window. A throttled heartbeat now skips while a commit is in flight, since that commit
+  already carries the stage's state. Terminal, error, initial, and forced step commits still queue
+  and land: they carry state no later heartbeat would repair.
+
 - A GRPO or OPD run whose rollout actor exited hard no longer wedges on a paid GPU until the
   wall-time limit. The actor's `os._exit` bypassed verl's own prompt handler, so the prompt
   stayed `running` forever and verl's unbounded replay poll waited on it at ~0% utilization. The

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.52
+Version: 1.2.58
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/core/spec.py

@@ -27,19 +27,7 @@ from uuid import UUID
 
 from flash._internal.diagnostics import SECRET_ENV_KEYS_ENV
 from flash.core.catalog import DEFAULT_MODEL, normalize_algorithm
-
-# `DROPPED_TOP_LEVEL_KEYS` keeps its historical private alias rather than being renamed at its 5 call
-# sites: `flash.runner` re-exports it as a package attribute and `preparation.py` reads it back through
-# `_runner()`, which is the seam the tests patch. Renaming it is a separate change from moving it, and
-# doing both at once would put a live recovery path's patch points inside a refactor whose whole claim
-# is that nothing behavioral moved.
-from flash.core.spec_persistence import (
-    DROPPED_TOP_LEVEL_KEYS as _DROPPED_TOP_LEVEL_KEYS,
-)
 from flash.core.spec_persistence import (
-    REMOVED_PERSISTED_TRAIN_KEYS,
-    announce_dropped_keys,
-    migrated_optimizer_batch,
     opt_float,
     opt_int,
     str_tuple,
@@ -654,10 +642,9 @@ class JobSpec:
         if not isinstance(data, dict):
             raise TypeError("job spec must be an object")
         allowed_top_level = {item.name for item in fields(cls)}
-        unknown_top_level = sorted(set(data) - allowed_top_level - _DROPPED_TOP_LEVEL_KEYS)
+        unknown_top_level = sorted(set(data) - allowed_top_level)
         if unknown_top_level:
             raise ValueError(f"job spec has unknown key(s): {', '.join(unknown_top_level)}")
-        announce_dropped_keys(data)
         env = data.get("environment") or {}
         # Reject stale payloads carrying a local `path`; worker only runs published env ids.
         if isinstance(env, dict) and env.get("path"):
@@ -665,12 +652,7 @@ class JobSpec:
                 "local environment paths are no longer supported; the worker only runs "
                 "published Freesolo environment ids"
             )
-        train = validated_section(
-            data,
-            "train",
-            {item.name for item in fields(TrainSpec)},
-            removed=REMOVED_PERSISTED_TRAIN_KEYS,
-        )
+        train = validated_section(data, "train", {item.name for item in fields(TrainSpec)})
         gpu = validated_section(data, "gpu", {item.name for item in fields(GpuSpec)})
         gpu_type, gpu_type_fallbacks = _parse_persisted_gpu_types(gpu)
         provider, providers = validated_persisted_providers(gpu, gpu_type, gpu_type_fallbacks)
@@ -679,9 +661,8 @@ class JobSpec:
             raise TypeError("project must be a string")
         project = require_project_id(project_raw) if project_raw.strip() else ""
         algorithm = normalize_algorithm(data.get("algorithm", cls.algorithm))
-        # one reading of the optimizer batch for both keys: the rollout spelling changed in 1.1.43
-        # and a persisted spec can still carry the old one.
-        batch_size, prompts_per_step = migrated_optimizer_batch(train, algorithm)
+        batch_size = opt_int(train.get("batch_size"))
+        prompts_per_step = opt_int(train.get("prompts_per_step"))
         return cls(
             model=data.get("model", cls.model),
             model_revision=_model_revision(data.get("model_revision", cls.model_revision)),

freesolo_flash_dev-1.2.58/flash/core/spec_persistence.py

@@ -0,0 +1,123 @@
+"""Decoding rules for PERSISTED job-spec records, as distinct from authored configuration.
+
+`JobSpec` serves four different contracts, and this module owns exactly one of them: reading back
+bytes that were already written to `~/.flash/runs/<run_id>.json`. That is not the same job as
+validating a config a user just wrote, and the two must not be confused:
+
+- authored configuration is parsed by `flash.schema.spec_from_dict`, which is STRICT. an unknown key
+  is a typo in a file the author can still fix, so it fails loudly.
+- a persisted record is parsed by `JobSpec.from_dict`, which applies the looser scalar coercions
+  here. It reads a record this build wrote, in this build's shape; an unknown key is fatal in both.
+
+Serialized-byte warning: `flash/runner/preparation.py::_preparation_digest` sha256-hashes the
+canonical JSON of `to_dict()` and `to_internal_dict()` output. Key ORDER is safe there
+(`sort_keys=True`), but key PRESENCE is not -- adding, dropping, or renaming a key in a decode path
+changes the digest and fails integrity validation for a run whose snapshot was already written.
+
+These functions take plain dicts and return plain dicts and scalars. That is the rule that keeps the
+module importable by `spec.py` rather than the other way round, and it is why `validated_section`
+takes its allowed-key set as an argument instead of reading `fields(TrainSpec)` itself.
+
+What is deliberately NOT here: `_coerce_wandb` and `_parse_persisted_gpu_types` are also called only
+from `JobSpec.from_dict`, so they belong to this contract by usage -- but they construct `WandbSpec`
+and reach `_validated_gpu_type`, which `__post_init__` needs too. Moving them would make this module
+import the dataclasses that import it. The boundary is therefore drawn at "persisted-decoding logic
+that does not depend on the spec types themselves".
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def validated_section(data: dict[str, Any], name: str, allowed: set[str]) -> dict[str, Any]:
+    """Read one nested block (``[train]``, ``[gpu]``) of a persisted record, rejecting unknown keys.
+
+    A missing block and an explicit null are the same state -- both mean "authored nothing" -- so both
+    normalize to empty rather than one of them raising. An unknown key is fatal.
+    """
+    section = data.get(name)
+    if section is None:
+        section = {}
+    if not isinstance(section, dict):
+        raise TypeError(f"{name} must be an object")
+    unknown = sorted(set(section) - allowed)
+    if unknown:
+        raise ValueError(f"{name} has unknown key(s): {', '.join(unknown)}")
+    return section
+
+
+def validated_persisted_providers(
+    gpu: dict[str, Any], gpu_type: str, gpu_type_fallbacks: tuple[str, ...]
+) -> tuple[str, tuple[str, ...]]:
+    """``(provider, providers)`` from a persisted ``[gpu]`` block, cross-checked against its classes.
+
+    ``GpuSpec.__post_init__`` runs the same both-set check but cannot replace this one: ``allow_empty``
+    here keys off whether the record CARRIED a ``providers`` key, and that presence is exactly what the
+    dataclass has already lost by the time it validates. An explicitly empty preference and an absent
+    one are different states in a stored record.
+    """
+    from flash.providers import PROVIDER_NAMES, validated_provider_preferences
+
+    provider = gpu.get("provider", "")
+    if not isinstance(provider, str):
+        raise TypeError("gpu.provider must be a string")
+    provider = provider.strip().lower()
+    providers = validated_provider_preferences(
+        gpu.get("providers", ()), allow_empty="providers" not in gpu
+    )
+    if provider and providers:
+        raise ValueError("gpu.provider and gpu.providers cannot both be set")
+    if provider or providers or gpu_type:
+        from flash.providers.base import providers_for
+
+        if provider and provider not in PROVIDER_NAMES:
+            raise ValueError(f"unknown gpu.provider {provider!r}")
+        # a hard provider pin must carry every acceptable class. provider preferences stay soft: an
+        # ineligible named provider contributes no candidate, while eligible named or unnamed
+        # configured providers remain available for failover.
+        for candidate in (gpu_type, *gpu_type_fallbacks):
+            if candidate and provider and provider not in providers_for(candidate):
+                raise ValueError(
+                    f"gpu.provider {provider!r} cannot provision gpu.type {candidate!r}"
+                )
+    return provider, providers
+
+
+def str_tuple(value: Any) -> tuple[str, ...]:
+    """Normalize a string-or-list knob to a tuple of strings; a bare string is one element, not iterated."""
+    if value is None:
+        return ()
+    if isinstance(value, str):
+        return (value,) if value else ()
+    return tuple(s for s in (str(x) for x in value) if s)
+
+
+def volume_gb(value: Any, default: int = 100) -> int:
+    """Parse volume size in GB; non-positive / non-numeric / missing values return default."""
+    if isinstance(value, bool):
+        # bool is an int subclass; reject to avoid int(True)==1 becoming a 1 GB volume.
+        return default
+    try:
+        gb = int(value)
+    except (TypeError, ValueError):
+        return default
+    return gb if gb > 0 else default
+
+
+def opt_int(value: Any) -> int | None:
+    """Parse optional int; rejects bools (bool is int subclass — int(True)==1 is a footgun)."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        raise TypeError(f"expected a number, got bool {value!r}")
+    return int(value)
+
+
+def opt_float(value: Any) -> float | None:
+    """Parse optional float; rejects bools (mirrors ``opt_int``)."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        raise TypeError(f"expected a number, got bool {value!r}")
+    return float(value)

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/cost/analytical.py

@@ -597,7 +597,15 @@ def _offline_preferred_gpu_shape(config: RunConfig) -> tuple[str, int, int, str,
                 quote[4] * quote[2] * sharded_step_seconds(config, quote[0], quote[2], quote[3])
             ),
         )
-    # preserve the existing unpinned diagnostic when no registered provider has a structural fit.
+    # every eligible provider has now been tried and none has a structural fit. dropping the
+    # restriction here to reach the unpinned `provider="auto"` diagnostic would rank the registered
+    # runpod pool regardless of credentials, quoting a shape this plane cannot rent -- the same
+    # defect the eligibility filter above exists to prevent, reintroduced at the last step. that
+    # quote would then pass affordability and only fail once live allocation runs, after the run is
+    # recorded. re-raise the eligible set's own failure instead, which names the real constraint.
+    if eligible:
+        return _offline_gpu_shape(replace(config, provider=eligible[0], providers=()))
+    # no provider is registered at all (an empty registry): keep the historical unpinned answer.
     return _offline_gpu_shape(replace(config, providers=()))
 
 
@@ -629,11 +637,18 @@ def _offline_gpu_shape(config: RunConfig) -> tuple[str, int, int, str, float]:
         providers_for,
         rentable_gpu_counts,
     )
-    from flash.providers.fit_errors import vram_fit_error_message, vram_knob_advice
 
     provider = config.provider if config.provider != "auto" else "auto"
     if config.gpu_type:
-        names = (canonical_gpu(config.gpu_type),)
+        # rank every class the author declared acceptable, not just the head. `allocate()` cost-ranks
+        # the whole ordered set, so quoting the head alone prices a shape the run may never be given
+        # -- an authored ["B200", "H100"] quotes 3x the H100 the allocator would actually rent, and
+        # the submit-time affordability precheck refuses an affordable run on that inflated number.
+        names = tuple(
+            dict.fromkeys(
+                canonical_gpu(name) for name in (config.gpu_type, *config.gpu_type_fallbacks)
+            )
+        )
     elif provider == "auto":
         # `auto` ranks the RunPod pool: it is the default substrate, and the allocator only reaches a
         # lambda-only class after the cheaper runpod classes exhaust, so quoting one here would name
@@ -700,43 +715,79 @@ def _offline_gpu_shape(config: RunConfig) -> tuple[str, int, int, str, float]:
                 )
             )
     if not ranked:
-        # a pinned class is blocked by the class itself, so name it -- the pool-wide message would
-        # report the widest validated shape, which is not hardware this quote was ever allowed to
-        # use. `_wider_shape_remedy` searches only `names`, already narrowed to the pin, so the
-        # `--gpus N` clause it appends can never name a shape the pin forbids. an unpinned run
-        # falls through to the pool-wide message, which reports the count that would fit.
-        remedy = _wider_shape_remedy(config, need, names)
-        if config.gpu_type:
-            info = GPU_INFO[canonical_gpu(config.gpu_type)]
-            raise ValueError(
-                f"exact GPU {info.name!r} cannot fit this run: it requires at least {need} GB"
-                + (
-                    remedy
-                    or _catalog_check_remedy(config, need, names)
-                    or f". {vram_knob_advice(config.method).capitalize()}."
-                )
-            )
-        raise ValueError(
-            vram_fit_error_message(
-                config.method,
-                need,
-                requested_gpu_count=ceiling,
-                effective_gpu_count=safe_gpu_count,
-                max_gpu_count=auto_cap,
-                gpu_names=names,
-                providers=None if provider == "auto" else (provider,),
-                # same rule the ranking loop rejected shapes with, so the advice cannot name a
-                # width the retry will not launch on.
-                executed_width=lambda count: executed_gpu_count(config, count),
-                # an offline quote does not know the configured fleet, so it cannot claim that
-                # dropping a provider pin would make a wider shape purchasable.
-                widenable_without_pin=None,
-            )
+        _raise_no_fitting_shape(
+            config,
+            need,
+            names,
+            provider=provider,
+            ceiling=ceiling,
+            safe_gpu_count=safe_gpu_count,
+            auto_cap=auto_cap,
         )
     _cost, count, _combined, _per_card, gpu, hourly = min(ranked)
     return gpu, need, count, provider, hourly
 
 
+def _raise_no_fitting_shape(
+    config: RunConfig,
+    need: float,
+    names: tuple[str, ...],
+    *,
+    provider: str,
+    ceiling: int | None,
+    safe_gpu_count: int,
+    auto_cap: int,
+) -> None:
+    """Reject a run no rentable shape fits, naming what the quote was allowed to rank."""
+    from flash.providers.base import GPU_INFO, canonical_gpu
+    from flash.providers.fit_errors import vram_fit_error_message, vram_knob_advice
+
+    # a pinned class is blocked by the class itself, so name it -- the pool-wide message would
+    # report the widest validated shape, which is not hardware this quote was ever allowed to
+    # use. `_wider_shape_remedy` searches only `names`, already narrowed to the pin, so the
+    # `--gpus N` clause it appends can never name a shape the pin forbids. an unpinned run
+    # falls through to the pool-wide message, which reports the count that would fit.
+    remedy = _wider_shape_remedy(config, need, names)
+    if config.gpu_type:
+        # name every acceptable class, not just the head: with fallbacks authored, reporting the
+        # head alone tells the user to fix a class that may not be the one that failed, and
+        # hides that the whole declared set was ranked and rejected.
+        declared = tuple(
+            GPU_INFO[canonical_gpu(name)].name
+            for name in (config.gpu_type, *config.gpu_type_fallbacks)
+        )
+        label = (
+            repr(declared[0])
+            if len(declared) == 1
+            else "none of " + ", ".join(repr(name) for name in declared)
+        )
+        raise ValueError(
+            f"exact GPU {label} cannot fit this run: it requires at least {need} GB"
+            + (
+                remedy
+                or _catalog_check_remedy(config, need, names)
+                or f". {vram_knob_advice(config.method).capitalize()}."
+            )
+        )
+    raise ValueError(
+        vram_fit_error_message(
+            config.method,
+            need,
+            requested_gpu_count=ceiling,
+            effective_gpu_count=safe_gpu_count,
+            max_gpu_count=auto_cap,
+            gpu_names=names,
+            providers=None if provider == "auto" else (provider,),
+            # same rule the ranking loop rejected shapes with, so the advice cannot name a
+            # width the retry will not launch on.
+            executed_width=lambda count: executed_gpu_count(config, count),
+            # an offline quote does not know the configured fleet, so it cannot claim that
+            # dropping a provider pin would make a wider shape purchasable.
+            widenable_without_pin=None,
+        )
+    )
+
+
 def _allocation_quote_shape(config: RunConfig, allocation) -> tuple[str, int, int, str, float]:
     """The (gpu, need, count, provider, per-card rate) billed against a SELECTED live candidate.
 

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/cost/spec.py

@@ -213,6 +213,11 @@ def runconfig_from_spec(spec) -> RunConfig:
         # and a caller that predates the preference has no such attribute to offer.
         providers=getattr(g, "providers", ()) or (),
         gpu_type=g.type,
+        # the rest of an ordered `[gpu] type` list. allocation cost-ranks every acceptable class, so
+        # dropping these here quotes the head alone and the affordability precheck can refuse a run
+        # whose cheaper authored fallback it would really have rented. read defensively for the same
+        # reason as `providers` above: this is not always a GpuSpec.
+        gpu_type_fallbacks=tuple(getattr(g, "type_fallbacks", ()) or ()),
         model_revision=spec.model_revision,
         disk_gb=float(getattr(g, "disk_gb", 0.0) or 0.0),
         # deliberately NOT spec.authored_gpu_count: this asks whether a shape has been RESOLVED

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/cost/types.py

@@ -61,6 +61,12 @@ class RunConfig:
     sft_retained_examples: int | None = None
     # appended for the same positional-constructor reason as sft_retained_examples above.
     providers: tuple[str, ...] = ()
+    # the remaining classes an ordered `[gpu] type` list accepts, after gpu_type takes the head.
+    # allocation cost-ranks the whole acceptable set, so quoting the head alone prices a shape the
+    # run may never be given: an authored ["B200", "H100"] is quoted on B200 while allocate() would
+    # rent the cheaper H100, and the affordability precheck runs on that inflated estimate.
+    # appended for the same positional-constructor reason as the two fields above.
+    gpu_type_fallbacks: tuple[str, ...] = ()
 
     def __post_init__(self) -> None:
         object.__setattr__(self, "method", normalize_algorithm(self.method))
@@ -87,6 +93,24 @@ class RunConfig:
             if prov != "auto" and prov not in providers_for(exact):
                 raise ValueError(f"provider {prov!r} cannot provision gpu_type {exact!r}")
         object.__setattr__(self, "gpu_type", exact)
+        # canonicalized and validated exactly like the head above, so a quote cannot rank a class the
+        # spec layer would have rejected. deduped against the head and each other because the
+        # ranking loop would otherwise price the same class twice.
+        fallbacks: list[str] = []
+        for entry in self.gpu_type_fallbacks or ():
+            if not isinstance(entry, str):
+                raise TypeError("gpu_type_fallbacks entries must be strings")
+            name = canonical_gpu(entry)
+            info = GPU_INFO.get(name)
+            if info is None or not info.validated:
+                raise ValueError(
+                    f"gpu_type_fallbacks entry {name!r} must name an active validated GPU class"
+                )
+            if name != exact and name not in fallbacks:
+                fallbacks.append(name)
+        if fallbacks and not exact:
+            raise ValueError("gpu_type_fallbacks requires gpu_type")
+        object.__setattr__(self, "gpu_type_fallbacks", tuple(fallbacks))
         if not isinstance(self.model_revision, str):
             raise TypeError("model_revision must be a string")
         object.__setattr__(self, "model_revision", self.model_revision.strip())

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/engine/worker/io/heartbeat.py

@@ -97,6 +97,19 @@ _HB_UPLOAD_LOCK = threading.Lock()
 # Terminal/error commits wait longer — no later heartbeat can repair them.
 _HB_UPLOAD_LOCK_TIMEOUT_S = 30.0
 _HB_CRITICAL_UPLOAD_LOCK_TIMEOUT_S = 120.0
+# True while a thread holds _HB_UPLOAD_LOCK and is inside the HF commit. Read under _HB_LOCK.
+#
+# The throttle clock (_HB_LAST_UPLOAD) only advances AFTER a commit lands, so during a slow HF
+# commit every other throttled caller still computes upload_due=True, walks into
+# _HB_UPLOAD_LOCK.acquire() and blocks there for up to the acquire timeout -- 30s, or 120s on a
+# critical stage. That stalls the liveness daemon, and `liveness_heartbeat`'s join delays leaving
+# the wrapped stage by the same window.
+#
+# A marker rather than claiming the clock before the upload: an optimistic claim has to be rolled
+# back when the commit fails, and the rollback has to decide whether a NEWER heartbeat already
+# moved the clock (the reason the old code carried a claim sequence). This says only "a commit is
+# in flight right now", is cleared in a finally, and so cannot strand the throttle on a failure.
+_HB_UPLOAD_IN_FLIGHT = False
 # retain at least one metric row per second across the 900s training heartbeat throttle window.
 GRPO_METRIC_HISTORY_LIMIT = 1024
 
@@ -127,6 +140,12 @@ def _console_heartbeat_snapshot(
     return json.dumps(console_payload)
 
 
+def _set_upload_in_flight(value: bool) -> None:
+    """Mark whether an HF heartbeat commit is running. Call with _HB_LOCK held."""
+    global _HB_UPLOAD_IN_FLIGHT
+    _HB_UPLOAD_IN_FLIGHT = value
+
+
 def _heartbeat_upload_due(
     stage: str,
     *,
@@ -140,6 +159,22 @@ def _heartbeat_upload_due(
     """return eligibility from committed throttle state while _HB_LOCK is held."""
     if initial or _is_critical_stage(stage):
         return True
+    # a commit is already in flight and will advance the throttle clock when it lands. an unforced
+    # ping would only queue behind it on _HB_UPLOAD_LOCK, publish a snapshot older than the one
+    # being written, and block its caller for the acquire timeout -- 30s, or 120s on a critical
+    # stage. that is what stalls the liveness daemon, whose `join` then delays leaving the wrapped
+    # stage by the same window. skipping loses nothing: the in-flight commit carries this stage.
+    #
+    # three exemptions, all above or inside this check:
+    #   - initial and critical stages return True earlier: a terminal or error snapshot has no later
+    #     heartbeat to repair it, so it must queue and land.
+    #   - `force` is excluded here because it carries billing state. a forced commit marks a
+    #     DISTINCT completed step, and a cancel bills the last step a heartbeat recorded, so
+    #     dropping one to avoid a wait could under-report a step the run really finished.
+    # the daemon's own pings are never forced (liveness and keepalive both leave it False), so this
+    # exemption does not reopen the stall it fixes.
+    if _HB_UPLOAD_IN_FLIGHT and not force:
+        return False
     if _w._HB_TERMINAL_ONLY:
         return (
             _w._HB_LAST_UPLOAD == 0.0 or (now - _w._HB_LAST_UPLOAD) >= _HB_TERMINAL_ONLY_INTERVAL_S
@@ -234,12 +269,18 @@ def heartbeat(
                     up = f"/tmp/.hb-upload-{os.getpid()}-{threading.get_ident()}.json"
                     with open(up, "w") as f:
                         f.write(snapshot)
+                    with _HB_LOCK:
+                        _set_upload_in_flight(True)
                     try:
                         if initial:
                             committed = _w.hf_upload_file(up, "heartbeat.json", required=True)
                         else:
                             committed = _w.hf_upload_file(up, "heartbeat.json")
                     finally:
+                        # cleared before the throttle clock is advanced below, and in a finally so a
+                        # raising upload cannot leave every later heartbeat permanently skipped.
+                        with _HB_LOCK:
+                            _set_upload_in_flight(False)
                         with contextlib.suppress(OSError):
                             os.remove(up)
                     if committed is False:

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/engine/worker/perf/__init__.py

@@ -5,7 +5,7 @@ The fla/tilelang/cudart fixup group stays IN THIS MODULE on purpose: tests monke
 through the patched module globals. Do NOT move this group into a submodule.
 
 NOTE: ``tests/test_worker_stack.py`` reads THIS file as TEXT to assert the fla SHA /
-TILELANG_PIN stay in lockstep with WORKER_DEPS / Dockerfile.worker.
+TILELANG_PIN stay in lockstep with Dockerfile.worker.
 """
 
 from __future__ import annotations
@@ -315,7 +315,7 @@ def _ensure_fla_fastpath_on_hopper() -> None:
             return False
         return rc == 0
 
-    # 0.1.12 double-registers the TVM-FFI runtime -> `import tilelang` aborts; keep in lockstep with WORKER_DEPS / Dockerfile.
+    # 0.1.12 double-registers the TVM-FFI runtime -> `import tilelang` aborts; keep in lockstep with Dockerfile.worker.
     TVM_FFI_PIN = "0.1.11"
     TILELANG_PIN = "0.1.11"
 
@@ -334,7 +334,7 @@ def _ensure_fla_fastpath_on_hopper() -> None:
         fla_ok = True
         if not (_have("fla") and _have("fla.modules")):
             _remove_fla_from_disk()
-            # Pinned commit — keep in lockstep with WORKER_DEPS / Dockerfile.worker.
+            # Pinned commit — keep in lockstep with Dockerfile.worker.
             fla_ok = _pip(
                 "--no-deps",
                 "git+https://github.com/fla-org/flash-linear-attention.git"

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/envs/identity.py

@@ -46,7 +46,7 @@ class GitHubUnavailableError(GitHubTransientError):
 
 
 class GitHubPermanentError(RuntimeError):
-    """GitHub answered, and the answer will not change on a retry: a 404 or a 422.
+    """GitHub answered, and the answer will not change on a retry: 401, 403, 404, or 422.
 
     Deliberately NOT a GitHubTransientError. Waiting is the right response to a blip and the wrong
     response to a typo, and the two are indistinguishable once both are a bare RuntimeError -- which
@@ -57,6 +57,10 @@ class GitHubPermanentError(RuntimeError):
     read, rather than leaking its existence with a 403. Both are permanent for this caller: no
     amount of retrying makes an unreadable ref resolvable, and the fix (check the name, or grant the
     token access) is the user's either way. The message carries GitHub's own text so it names which.
+
+    The credential codes reach here only after the rate-limit shapes (429, and the 403 that says
+    rate limit) have been claimed as transient, so a 401 or a surviving 403 is a token this plane
+    cannot fix by waiting: invalid, expired, or missing the scope. Same user-side fix as a typo.
     """
 
 

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/envs/loader.py

@@ -285,9 +285,13 @@ def _urlopen(
                 ) from exc
             # 404 (no such repo, or one the token cannot read) and 422 (a ref that does not exist,
             # e.g. "No commit found for SHA: main" on a master-default repo) are settled answers.
+            # 401 and a non-rate-limit 403 join them: the rate-limit shapes are already claimed
+            # above, so what is left is a token this plane cannot fix by waiting -- invalid,
+            # expired, or lacking the scope. deferring those rents a GPU to rediscover the same
+            # credential error on the worker.
             # Typed so the submit-time pin can fail closed on them while still deferring a blip;
             # every other code keeps the untyped RuntimeError it has always raised.
-            if exc.code in (404, 422):
+            if exc.code in (401, 403, 404, 422):
                 raise GitHubPermanentError(
                     f"GitHub environment request failed ({exc.code}): {body[:500]}"
                 ) from exc

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/providers/_lifecycle/bootstrap.py

@@ -257,15 +257,31 @@ def hf_upload(
         return False
 
 
-def _upload_console_snapshot(payload: dict, console: str, mode: str, extra: str = "") -> bool:
-    """Upload one console snapshot from an isolated process."""
-    tail_path = console + ".tail"
+def _upload_console_snapshot(
+    payload: dict, console: str, mode: str, extra: str = "", final: bool = False
+) -> bool:
+    """Upload one console snapshot from an isolated process.
+
+    The periodic snapshot and the terminal one never share a scratch file or a repo destination.
+    Reaping the periodic child at teardown is best-effort, so one killed mid-write would otherwise
+    truncate the scratch file the terminal snapshot is uploading, or overwrite the terminal artifact
+    itself -- losing the failure detail the control plane reads, including the wall-clock-cap marker.
+
+    The terminal snapshot keeps the canonical ``console_<mode>.txt`` the control plane reads; the
+    periodic one takes the attempt-scoped name it reads separately, matching the serverless handler,
+    so a retry cannot overwrite the attempt that reproduced the failure. This module can never
+    import flash, so the format is spelled out here and pinned against
+    ``flash.adapters.artifacts.attempt_scoped_artifact_name`` by test rather than by coincidence --
+    a name the reader does not expect looks exactly like a worker that uploaded nothing.
+    """
+    kind = "" if final else f"_attempt{payload.get('attempt', 0)}"
+    tail_path = f"{console}{kind}.tail"
     tail = _read_console_tail(console, 64_000, secrets=_payload_secrets(payload))
     if extra:
         tail += extra
     with open(tail_path, "w", encoding="utf-8", errors="replace") as f:
         f.write(_safe_detail(tail, 64_000, secrets=_payload_secrets(payload)))
-    return hf_upload(payload, tail_path, f"console_{mode}.txt")
+    return hf_upload(payload, tail_path, f"console_{mode}{kind}.txt")
 
 
 def _console_upload_loop(
@@ -453,7 +469,7 @@ def _upload_console_tail_bounded(
     context = multiprocessing.get_context("spawn")
     process = context.Process(
         target=_upload_console_snapshot,
-        args=(payload, console, mode, extra),
+        args=(payload, console, mode, extra, True),
         daemon=True,
     )
     process.start()

{freesolo_flash_dev-1.2.52 → freesolo_flash_dev-1.2.58}/flash/providers/_lifecycle/bootstrap_console.py

@@ -77,8 +77,13 @@ def _run_console_upload_loop(console: str, interval_s: float, stop_upload, *, up
     prevents a flapping run from turning the poll cadence into the commit cadence.
 
     Before any heartbeat commits, one additional setup snapshot lands before the fixed 3000-second
-    teardown. It does not enter the steady rate: the first committed heartbeat moves the deadline to
-    the normal interval. failed uploads change no upload state, so the next poll retries.
+    teardown. It does not enter the steady rate: the first committed heartbeat promotes the deadline
+    to the normal interval, so a run that commits before 600 seconds never spends that write. the
+    promotion is skipped while ``due`` holds, so it only ever moves a deadline that has not arrived.
+    A 600-second deadline that already expired with nothing committed still spends its snapshot --
+    the startup evidence it exists for -- and a failed upload on that same poll leaves the deadline
+    due, so the next poll retries rather than waiting out the hour. Once promoted the deadline is the
+    hourly interval, so the guard cannot re-fire early and needs no separate first-commit flag.
     """
     poll_s = min(_CONSOLE_UPLOAD_POLL_S, interval_s)
     due_s = min(_CONSOLE_UPLOAD_FIRST_SNAPSHOT_S, interval_s)
@@ -87,14 +92,13 @@ def _run_console_upload_loop(console: str, interval_s: float, stop_upload, *, up
     while not stop_upload.wait(poll_s):
         since += poll_s
         cursor, eof, staged, beats = _console_progress(console, max(cursor, 0))
-        had_committed = ever
         ever = ever or bool(staged)
-        if ever and not had_committed and sent >= 0:
+        due = since >= due_s
+        if ever and not due:
             due_s = interval_s
         made_progress = staged if ever else beats
         armed = armed or bool(made_progress)
         quiet = 0.0 if made_progress else quiet + 1
-        due = since >= due_s
         cap = armed and quiet >= _CONSOLE_UPLOAD_QUIET_POLLS
         wedged = cap and spent < _CONSOLE_UPLOAD_CREDITS and not due
         if eof == sent or not (due or wedged):

freesolo_flash_dev-1.2.58/flash/providers/_lifecycle/host/__init__.py

@@ -0,0 +1,5 @@
+"""Standalone host-side helper programs shipped in the instance-bootstrap capsule.
+
+Each module here runs as its own `python3 <file>` process on the rented box, NOT as an import of
+the flash package. They must stay stdlib-only and self-contained.
+"""