freesolo-flash-dev 0.2.38__tar.gz → 0.2.40__tar.gz

{freesolo_flash_dev-0.2.38 → freesolo_flash_dev-0.2.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 0.2.38
+Version: 0.2.40
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-0.2.38 → freesolo_flash_dev-0.2.40}/docker/kernel_fingerprint.py

@@ -143,13 +143,13 @@ def collect_inputs(
 
     Each value comes from where the IMAGE actually gets it:
       * pins / FROM / causal-conv1d from Dockerfile.worker (what the image is built from),
-      * default chalk spec textually from deps.py (what the bake warms),
+      * default chalk spec textually from _worker.py (what the bake warms),
       * FA2/FA3 from worker-image.yml's build-args (overridable via fa2_spec/fa3_spec for the
         resolved-build-arg case), and file hashes for the warmup/handler sources.
     base_inputs_partial does NOT yet include fp_cache; compute_fingerprints folds it in.
     """
     dockerfile = (root / "Dockerfile.worker").read_text()
-    deps = (root / "flash" / "providers" / "runpod" / "train" / "deps.py").read_text()
+    worker_pkg = (root / "flash" / "providers" / "_worker.py").read_text()
     worker_image_yml = (root / ".github" / "workflows" / "worker-image.yml").read_text()
 
     specs = _pip_stack_specs(dockerfile)
@@ -168,7 +168,9 @@ def collect_inputs(
         raise ValueError(
             "kernel_fingerprint: fla spec missing or not pinned to a 40-char commit sha"
         )
-    chalk = _python_string_constant(deps, "DEFAULT_CHALK_SPEC", "deps.py DEFAULT_CHALK_SPEC")
+    chalk = _python_string_constant(
+        worker_pkg, "DEFAULT_CHALK_SPEC", "_worker.py DEFAULT_CHALK_SPEC"
+    )
 
     cache_inputs = {
         "from_image": from_image,

{freesolo_flash_dev-0.2.38 → freesolo_flash_dev-0.2.40}/flash/cli/render.py

@@ -359,6 +359,8 @@ def _color_json(obj, depth: int) -> str:
 
 def _hide_provider_metadata(obj):
     """Styled CLI details are for humans; keep backend provider names out of that view."""
+    from flash.providers import PROVIDER_NAMES
+
     if isinstance(obj, dict):
         return {
             k: _hide_provider_metadata(v)
@@ -367,7 +369,7 @@ def _hide_provider_metadata(obj):
         }
     if isinstance(obj, list):
         return [_hide_provider_metadata(v) for v in obj]
-    if isinstance(obj, str) and obj.lower() in {"runpod", "lambda"}:
+    if isinstance(obj, str) and obj.lower() in PROVIDER_NAMES:
         return "managed"
     return obj
 

{freesolo_flash_dev-0.2.38 → freesolo_flash_dev-0.2.40}/flash/cost/analytical.py

@@ -98,7 +98,6 @@ def seconds_per_step(config: RunConfig, gpu: str) -> float:
     reward_s = math.ceil(completions / REWARD_CONCURRENCY) * latency  # ceil: a partial wave still costs one latency
     return gen_s + reward_s + update_s
 
-
 def sft_seconds_for_tokens(config: RunConfig, gpu: str, train_tokens: float) -> float:
     """SFT steady-state wall time for an actual token count on ``gpu``."""
     n = config.normalized()
@@ -108,7 +107,7 @@ def sft_seconds_for_tokens(config: RunConfig, gpu: str, train_tokens: float) ->
     return flops / (peak * MFU_SFT_TRAIN)
 
 
-def select_gpu(config: RunConfig) -> tuple[str, int]:
+def select_gpu(config: RunConfig, *, max_wall_seconds: float = 0.0) -> tuple[str, int]:
     """(chosen GPU class, required VRAM GB): the cheapest fitting class for the cost.
 
     Uses ``pick_gpu``, which (unlike the submit-time allocator) intentionally stays gate-free —
@@ -122,7 +121,7 @@ def select_gpu(config: RunConfig) -> tuple[str, int]:
         train=config.train_knobs(),
         thinking=config.thinking,
     )
-    gpu = pick_gpu(need, provider=config.provider)
+    gpu = pick_gpu(need, provider=config.provider, max_wall_seconds=max_wall_seconds)
     return gpu, need
 
 
@@ -153,10 +152,26 @@ def _notes(config: RunConfig, raw_train_s: float, wall_capped: bool, cap_s: floa
 
 def estimate_cost(config: RunConfig, *, wall_cap_s: float = DEFAULT_WALL_CAP_S) -> CostEstimate:
     """Deterministic pre-flight cost calculation."""
-    gpu, need = select_gpu(config)
-    hourly = gpu_hourly_usd(gpu, provider=config.provider)
-    # Mirror the runner's max(60, max_wall_seconds) floor so elapsed wall time is not undercounted.
+    # Billing cap: mirror the runner's max(60, max_wall_seconds) floor so a sub-60s cap isn't underpriced.
     cap_s = max(60.0, float(config.max_wall_seconds)) if config.max_wall_seconds is not None else wall_cap_s
+    # Vast market duration filter: price against offers that outlast the run, using the SAME semantics
+    # ``usable_offers`` applies at LAUNCH (not the 60s-floored billing cap_s) — a non-positive wall means
+    # NO filter, a positive one is floored at 60s by usable_offers itself:
+    #   None -> the 24h spec default the run runs under (== DEFAULT_WALL_CAP_S);
+    #   > 0  -> that wall;   <= 0 -> 0.0 (no filter, exactly like launch).
+    if config.max_wall_seconds is None:
+        market_wall_s = wall_cap_s
+    elif config.max_wall_seconds > 0:
+        market_wall_s = float(config.max_wall_seconds)
+    else:
+        market_wall_s = 0.0
+    gpu, need = select_gpu(config, max_wall_seconds=market_wall_s)
+    # Quote the SAME VRAM-floored Vast market pick_gpu selected under (min_vram_gb=need): without the
+    # floor the rate lookup searches from the smallest managed class, letting cheap small-card offers
+    # crowd a high-VRAM selection off the limited page -> it silently falls back to the static rate.
+    hourly = gpu_hourly_usd(
+        gpu, provider=config.provider, max_wall_seconds=market_wall_s, min_vram_gb=need
+    )
 
     setup = setup_seconds(config)
     sps = seconds_per_step(config, gpu)

freesolo_flash_dev-0.2.40/flash/cost/facts.py

@@ -0,0 +1,161 @@
+"""Static lookup facts for the cost model: GPU price/VRAM/compute + cheapest-fit
+selection, model size/quant, and reward-grader latency. Pure tables + accessors."""
+
+from __future__ import annotations
+
+from flash.catalog import MODELS
+from flash.providers.base import GPU_INFO, GpuClass, providers_for
+
+GPU_COMPUTE_TFLOPS: dict[str, float] = {
+    # A10: 125 TFLOPS dense bf16 tensor (NVIDIA spec); Lambda-only 24 GB class, else defaults to 100.
+    "A10": 125.0,
+    "RTX 4090": 165.0,
+    "RTX 5090": 210.0,
+    "A100 PCIe": 312.0,
+    "A100 SXM": 312.0,
+    # A100 SXM 40GB: same SMs/tensor cores as the 80GB A100 SXM, less HBM only.
+    # Without this, 33-40 GB Lambda/Vast quotes fall back to _DEFAULT_TFLOPS.
+    "A100 SXM 40GB": 312.0,
+    "H100": 990.0,
+    # H200: same SMs/tensor cores as H100, more HBM only.
+    "H200": 990.0,
+    "RTX Pro 6000": 250.0,
+    # B200: 2.25 PFLOPS bf16 dense (NVIDIA spec); prevents ~10x cost over-estimate vs _DEFAULT_TFLOPS.
+    "B200": 2250.0,
+}
+_DEFAULT_TFLOPS = 100.0
+
+
+def gpu_tflops(name: str) -> float:
+    """Peak bf16 tensor TFLOPS for a managed GPU class."""
+    return GPU_COMPUTE_TFLOPS.get(name, _DEFAULT_TFLOPS)
+
+
+def gpu_hourly_usd(
+    name: str, provider: str | None = None, max_wall_seconds: float = 0.0, min_vram_gb: int = 0
+) -> float:
+    """Representative $/hr for a class, on ``provider`` when given.
+
+    When ``provider`` is ``lambda`` or ``vast`` and the class is offered there, price it through that
+    provider's pricing module (live with a static fallback); otherwise use the RunPod static rate.
+
+    ``max_wall_seconds`` (>0) is threaded into the Vast live market so a duration-bound quote prices
+    against offers that outlast the run, not a short-lived one filtered out at launch.
+
+    ``min_vram_gb`` (>0) floors the Vast market search at the job's required VRAM — the SAME floor
+    ``pick_gpu`` selected under — so a high-VRAM class isn't crowded off the price-sorted page and
+    misquoted on the static fallback (selection/quote parity).
+    """
+    info = GPU_INFO.get(name)
+    if info is None:
+        raise KeyError(f"unknown GPU class {name!r}")
+    p = (provider or "").strip().lower()
+    if p == "lambda" and info.lambda_name:
+        from flash.providers import get_provider
+
+        return get_provider("lambda").hourly_rate(name)
+    if p == "vast" and info.vast_name:
+        # Vast is a live market whose rates differ materially from RunPod's static ones, so price a
+        # provider="vast" quote through the Vast pricing module (live + static fallback).
+        from flash.providers.vast.pricing import hourly_rate
+
+        return hourly_rate(name, max_wall_seconds=max_wall_seconds, min_vram_gb=min_vram_gb)
+    return info.hourly_usd
+
+
+def gpu_vram_gb(name: str) -> int:
+    info = GPU_INFO.get(name)
+    if info is None:
+        raise KeyError(f"unknown GPU class {name!r}")
+    return info.vram_gb
+
+
+def pick_gpu(
+    required_vram_gb: int, *, provider: str | None = None, max_wall_seconds: float = 0.0
+) -> str:
+    """Cheapest GPU class that fits ``required_vram_gb``, ranked by static $/hr.
+
+    No pin; every fitting class is eligible, validated or not. NOTE this is intentionally
+    gate-free: the submit-time allocator restricts to the validated pool, so the
+    actually-provisioned class can be pricier than the one priced here. ``provider`` restricts
+    candidates to what it can provision. ``max_wall_seconds`` (>0) prices the Vast market against
+    offers that outlast the run, so a long-run quote doesn't SELECT a class on the strength of a
+    short-lived offer that won't survive to launch.
+    """
+
+    def _selectable(g: GpuClass) -> bool:
+        return provider in (None, "auto") or provider in providers_for(g.name)
+
+    candidates = [g for g in GPU_INFO.values() if g.vram_gb >= required_vram_gb and _selectable(g)]
+    if not candidates:
+        raise ValueError(f"no GPU class fits >= {required_vram_gb} GB")
+    # Rank by the rate on the REQUESTED provider, not the RunPod nominal. For Vast, fetch the live offer
+    # map ONCE (a duration-bound query bypasses the per-call cache, so pricing per candidate would fire N
+    # identical market fetches) and, when reachable, restrict to classes that actually have a rentable
+    # offer under the wall cap and rank by LIVE price — else a cheaper class with no surviving offer gets
+    # selected/quoted on a static rate the launch path would never rent. Static fallback when offline.
+    if (provider or "").strip().lower() == "vast":
+        from flash.providers.vast.pricing import live_offer_rates
+
+        live = live_offer_rates(max_wall_seconds=max_wall_seconds, min_vram_gb=required_vram_gb)
+        rentable = [g for g in candidates if g.name in live] if live else []
+        if rentable:
+            candidates = rentable
+
+            def _rate(g: GpuClass) -> float:
+                return live[g.name]
+        else:
+
+            def _rate(g: GpuClass) -> float:
+                return g.hourly_usd
+    else:
+
+        def _rate(g: GpuClass) -> float:
+            return gpu_hourly_usd(g.name, provider=provider, max_wall_seconds=max_wall_seconds)
+
+    best = min(candidates, key=lambda g: (_rate(g), g.vram_gb, g.name))
+    return best.name
+
+
+def total_params_b(model_id: str) -> float:
+    """Total parameter count (billions) for a catalog model."""
+    info = MODELS.get(model_id)
+    if info is None:
+        raise ValueError(
+            f"unknown model {model_id!r}; cost estimation supports catalog models only "
+            f"({', '.join(MODELS)})"
+        )
+    return info.params_b
+
+
+def active_params_b(model_id: str) -> float:
+    """Active params per token (billions); falls back to total for dense models. Use for FLOPs, not VRAM."""
+    info = MODELS.get(model_id)
+    if info is None:
+        raise ValueError(
+            f"unknown model {model_id!r}; cost estimation supports catalog models only "
+            f"({', '.join(MODELS)})"
+        )
+    return info.active_params_b or info.params_b
+
+
+def model_quant(model_id: str) -> str:
+    """Quantization of the catalog entry; defaults to 'bf16'."""
+    info = MODELS.get(model_id)
+    return (info.quant or "bf16") if info is not None else "bf16"
+
+
+def download_weight_gb(model_id: str) -> float:
+    """Full bf16 checkpoint size in GB (2 bytes/param)."""
+    return total_params_b(model_id) * 2.0
+
+
+# ~1s mid-range default across grader types (regex ~0.01s to LLM judge ~3s).
+AVG_REWARD_SECONDS_PER_COMPLETION = 1.0
+
+
+def reward_seconds_per_completion(override: float | None = None) -> float:
+    """Per-completion reward latency (s): the explicit override, else the single average."""
+    if override is not None:
+        return max(0.0, override)
+    return AVG_REWARD_SECONDS_PER_COMPLETION

freesolo_flash_dev-0.2.40/flash/providers/__init__.py

@@ -0,0 +1,63 @@
+"""Pluggable GPU substrates.
+
+The training worker (``flash.engine.worker``) reads a JobSpec from the environment, pulls code
+from the HF dataset repo, and streams artifacts/heartbeats/metrics back to it. The provider
+owns pricing, provisioning, polling, cancellation, and teardown.
+
+  runpod      serverless Flash endpoints (always on)
+  lambda      Lambda Cloud GPU instances (instance-based complement; iff LAMBDA_API_KEY set)
+  vast        Vast.ai verified-datacenter containers (live-market complement; iff VAST_API_KEY set)
+
+This module is the registry: ``get_provider(name)`` / ``PROVIDER_NAMES``.
+``allocator.allocate`` iterates the active provider list below; ``available_providers`` narrows
+it to the ones configured on THIS control plane (Lambda/Vast are opt-in via their operator
+keys, so a box without them silently behaves exactly as the RunPod-only setup).
+"""
+
+from __future__ import annotations
+
+from functools import cache
+
+from flash.providers.base import Provider
+
+# Registry / iteration order only — NOT a selection preference. Allocation ranks candidates purely
+# by price (see allocator.allocate), so runpod/lambda/vast get no tie-break edge from this order.
+PROVIDER_NAMES: tuple[str, ...] = ("runpod", "lambda", "vast")
+
+# Instance-billed providers: they rent a VM/container that BILLS UNTIL TERMINATED, so they need the
+# generic instance-cleanup paths (retry teardown, gc-by-label) that endpoint-based RunPod doesn't.
+# Single source of truth — keep cleanup/realization sites pointed here so a new instance provider is
+# wired into every reaper by adding ONE name (not by hunting hardcoded ("lambda", "vast") tuples).
+INSTANCE_PROVIDERS: tuple[str, ...] = ("lambda", "vast")
+
+
+def get_provider(name: str) -> Provider:
+    """Return the ``Provider`` singleton for a registered name (raises on unknown)."""
+    return _get_provider((name or "").strip().lower())
+
+
+@cache
+def _get_provider(key: str) -> Provider:
+    if key == "runpod":
+        from flash.providers.runpod import PROVIDER
+
+        return PROVIDER
+    if key == "lambda":
+        from flash.providers.lambdalabs import PROVIDER
+
+        return PROVIDER
+    if key == "vast":
+        from flash.providers.vast import PROVIDER
+
+        return PROVIDER
+    raise KeyError(f"unknown provider {key!r} (known: {', '.join(PROVIDER_NAMES)})")
+
+
+def available_providers() -> tuple[str, ...]:
+    """Provider names whose credentials are present on this control plane."""
+    return tuple(n for n in PROVIDER_NAMES if get_provider(n).is_configured())
+
+
+def configured_providers() -> list[Provider]:
+    """The ``Provider`` objects available right now (see ``available_providers``)."""
+    return [get_provider(n) for n in available_providers()]

freesolo_flash_dev-0.2.40/flash/providers/_hf_artifacts.py

@@ -0,0 +1,177 @@
+"""Provider-neutral HF-artifact reading + heartbeat provenance.
+
+The control plane observes a remote worker purely through the artifacts it uploads to a private HF
+dataset repo: rate-limited text/JSON readers for ``heartbeat.json`` / ``error_*.txt`` / console logs,
+plus the provenance predicates that decide whether a heartbeat belongs to THIS attempt or a leftover
+prior one. None of this is provider-specific — RunPod, Lambda, and Vast all poll the same artifact
+shape — so it lives here in the shared kernel and no provider package imports another for it.
+
+The worker-side bootstrap (``_instance_bootstrap``) cannot import flash, so it re-implements the
+upload half; this module is the read half every poller shares.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+
+from flash.providers._poll import _attempt_int
+
+
+def make_hf_text_reader(hf_repo: str, path_in_repo: str, min_interval_s: float = 45.0):
+    """Rate-limited reader for an HF artifact; returns None until it exists or on any error."""
+    state = {"last": 0.0}
+
+    def read(force: bool = False) -> str | None:
+        if not hf_repo:
+            return None
+        if not force and time.time() - state["last"] < min_interval_s:
+            return None
+        state["last"] = time.time()
+        try:
+            from huggingface_hub import hf_hub_download
+
+            p = hf_hub_download(
+                hf_repo,
+                path_in_repo,
+                repo_type="dataset",
+                token=os.environ.get("HF_TOKEN"),
+                force_download=True,
+            )
+            with open(p) as f:
+                return f.read()
+        except Exception:
+            return None
+
+    return read
+
+
+def make_hf_heartbeat_reader(hf_repo: str, prefix: str, min_interval_s: float = 30.0):
+    """Rate-limited JSON reader for ``{prefix}/heartbeat.json`` on HF."""
+    text_reader = make_hf_text_reader(hf_repo, f"{prefix}/heartbeat.json", min_interval_s)
+
+    def read(force: bool = False) -> dict | None:
+        raw = text_reader(force=force)
+        if raw is None:
+            return None
+        try:
+            return json.loads(raw)
+        except (ValueError, TypeError):
+            return None
+
+    return read
+
+
+def heartbeat_reader_for(spec):
+    """The HF heartbeat reader for a run's spec (None when the run has no hf_repo)."""
+    hf_repo = spec.train.hf_repo
+    return make_hf_heartbeat_reader(hf_repo, f"{spec.phase}/{spec.run_id}") if hf_repo else None
+
+
+def error_artifact_name(phase: str, attempt) -> str:
+    """Worker error-artifact filename for a phase+attempt (mirrors the worker's error_artifact_name)."""
+    return f"error_{phase}_attempt{int(attempt or 0)}.txt"
+
+
+def make_hf_failure_detail_reader(
+    hf_repo: str,
+    prefix: str,
+    phase: str,
+    min_interval_s: float = 45.0,
+    attempt: int = 0,
+):
+    """Reader for worker-uploaded failure artifacts on HF (error/console txt); force-read after terminal failure."""
+    # Attempt-scoped to match the worker's error_artifact_name(mode, attempt).
+    err_name = error_artifact_name(phase, attempt)
+    error_reader = make_hf_text_reader(hf_repo, f"{prefix}/{err_name}", min_interval_s)
+    console_reader = make_hf_text_reader(
+        hf_repo, f"{prefix}/console_{phase}.txt", min_interval_s
+    )
+
+    def read(force: bool = False) -> str | None:
+        parts: list[str] = []
+        error_text = error_reader(force=force)
+        if error_text:
+            parts.append(f"--- {err_name} ---\n{error_text[-4000:]}")
+        console_text = console_reader(force=force)
+        if console_text:
+            parts.append(f"--- console_{phase}.txt ---\n{console_text[-4000:]}")
+        return "\n".join(parts) if parts else None
+
+    return read
+
+
+def _heartbeat_is_prior_attempt(hb: dict, launch_ts: float | None, current_attempt) -> bool:
+    """Positively attribute a heartbeat to a PRIOR (earlier) attempt: an explicit ``attempt`` differing
+    from ``current_attempt`` (definitive provenance — a MATCH proves THIS launch, so a lagging worker-host
+    clock cannot demote it), else a parseable ``ts`` predating this attempt's launch. Un-dateable (no
+    attempt AND no usable ts) -> False: mere absence of proof is never treated as a leftover. Truthy
+    ``launch_ts`` only (0.0 = unknown launch, uncomparable). Shared decision core of
+    ``worker_flagged_retriable`` (honor the retriable flag iff NOT prior) and
+    ``heartbeat_is_stale_prior_attempt`` (stale iff prior) — one edit site instead of two lockstep copies."""
+    hb_attempt = _attempt_int(hb.get("attempt"))
+    cur_attempt = _attempt_int(current_attempt)
+    if hb_attempt is not None and cur_attempt is not None:
+        return hb_attempt != cur_attempt
+    if launch_ts:
+        try:
+            ts = float(hb.get("ts"))
+        except (TypeError, ValueError):
+            ts = None
+        if ts is not None and ts < float(launch_ts):
+            return True
+    return False
+
+
+def worker_flagged_retriable(
+    heartbeat_reader, *, launch_ts: float | None = None, current_attempt: int | None = None
+) -> bool:
+    """True if the worker stamped ``retriable`` (a RetriableInfraError) in its last heartbeat — the
+    structured worker<->poller contract that replaces failure-detail parsing: ``retriable`` means
+    retry on a fresh worker. Forces a fresh read past the rate limit.
+
+    ``launch_ts`` / ``current_attempt``, when supplied, gate the flag to THIS attempt. The seed
+    heartbeat path is shared across retries, so a leftover ``retriable=True`` from attempt N-1 must
+    NOT override attempt N's own (non-retriable) failure marker — otherwise a deterministic
+    bootstrap/config error that fails BEFORE this attempt's worker emits any heartbeat would be
+    reported job_preempted and burn GPUs on an endless retry instead of failing fast. Only positive
+    prior-attempt evidence gates the flag: a ts that predates launch OR an explicit attempt mismatch.
+    With NEITHER arg the flag is honored ungated (back-compat for callers that don't date heartbeats)."""
+    if heartbeat_reader is None:
+        return False
+    hb = heartbeat_reader(force=True)
+    if not isinstance(hb, dict):
+        return False
+    if not bool(hb.get("retriable")):
+        return False
+    if launch_ts is None and current_attempt is None:
+        return True  # ungated: caller can't date the heartbeat -> preserve prior behavior
+    # Honor the retriable flag unless the heartbeat provably belongs to a PRIOR attempt.
+    return not _heartbeat_is_prior_attempt(hb, launch_ts, current_attempt)
+
+
+def heartbeat_is_stale_prior_attempt(
+    heartbeat_reader, *, launch_ts: float | None = None, current_attempt: int | None = None
+) -> bool:
+    """True ONLY when a heartbeat can be POSITIVELY attributed to a PRIOR (earlier) attempt — either it
+    carries an explicit ``attempt`` that differs from ``current_attempt``, OR a parseable ``ts`` that
+    predates THIS attempt's launch. Everything else returns False: no heartbeat, an empty/uninformative
+    heartbeat (no ts AND no attempt — e.g. ``{}``), a heartbeat matching this attempt, or one that
+    cannot be dated. The asymmetry is deliberate — we suppress a crash classification only on PROOF of
+    a leftover, never on mere absence of proof (an un-dateable heartbeat is NOT evidence of a prior run
+    and must not mask THIS attempt's deterministic crash).
+
+    The seed heartbeat path AND the seed-scoped ``error_<phase>.txt`` crash artifact are BOTH shared
+    across this seed's retries, so a prior attempt can leave either behind. When the latest heartbeat
+    provably belongs to an earlier attempt, the co-located error file is presumed leftover too — so a
+    dead-host poll on attempt N must NOT read that stale crash file as THIS attempt's DETERMINISTIC
+    failure (which would fail-fast a genuine host LOSS instead of retrying it on a fresh host). Gating
+    requires BOTH ``launch_ts`` and ``current_attempt``; without them a heartbeat cannot be dated, so
+    it is never called stale (conservative — keep the caller's existing classification)."""
+    if heartbeat_reader is None:
+        return False
+    hb = heartbeat_reader(force=True)
+    if not isinstance(hb, dict) or launch_ts is None or current_attempt is None:
+        return False
+    return _heartbeat_is_prior_attempt(hb, launch_ts, current_attempt)

{freesolo_flash_dev-0.2.38 → freesolo_flash_dev-0.2.40}/flash/providers/_instance.py

@@ -1,4 +1,4 @@
-"""Shared building blocks for the instance-based providers (e.g. Lambda)."""
+"""Shared building blocks for the instance-based providers (Lambda, Vast)."""
 
 from __future__ import annotations
 
@@ -6,7 +6,9 @@ import base64
 import hashlib
 import io
 import json
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Any, ClassVar
 
 # Bounded so the name is never truncated at launch — truncation desyncs the sweep-matched prefix.
 _MAX_NAME = 60
@@ -26,6 +28,14 @@ def run_label_prefix(run_id: str) -> str:
     return f"{base[: _PREFIX_BUDGET - 9]}-{h}"
 
 
+def label_matches_run(label: str, prefix: str) -> bool:
+    """True iff ``label`` belongs to the run whose prefix is ``prefix`` — an EXACT match, or the prefix
+    followed by the ``-s`` seed boundary. Boundary-anchored so ``flash-100`` never matches
+    ``flash-1000-...`` (or vice versa). The single label-ownership test every instance provider's sweep
+    and run-scoped teardown shares."""
+    return label == prefix or label.startswith(prefix + "-s")
+
+
 def instance_label(run_id: str, seed: int, attempt: int) -> str:
     """Instance name: run-derived so ``sweep_orphans`` can tell ours from anything else on the
     account, and bounded (via ``run_label_prefix``) so the provider never truncates it."""
@@ -45,6 +55,70 @@ def instance_label(run_id: str, seed: int, attempt: int) -> str:
     return f"{run_label_prefix(run_id)}-s{seed_s}-a{attempt_s}"
 
 
+@dataclass
+class InstanceJobHandle:
+    """Fields + (de)serialization common to every rent-a-box provider handle (Lambda, Vast).
+
+    Persisted in ``RunStatus.remote`` so any process can reattach/cancel (cf. ``base.JobHandle``). Each
+    provider subclass tags its ``provider`` and adds its own locator fields (via ``_extra_to_dict`` /
+    ``_extra_from_dict``); the shared ``to_dict``/``from_dict`` carry the common set so the serialized
+    shape stays byte-identical across providers. ``instance_id`` is the poll/destroy target, so unlike
+    every other field it has NO safe default — a missing/uncoercible one is a corrupt handle that must
+    fail with a clear, actionable error (not a bare KeyError/ValueError that crashes reattach/recovery).
+    """
+
+    instance_id: int | str
+    gpu: str
+    hourly_usd: float
+    attempt: int
+    started_ts: float
+
+    provider: ClassVar[str] = "instance"
+
+    @staticmethod
+    def _coerce_instance_id(raw: Any) -> Any:
+        """Provider-specific instance_id coercion (Vast=int, Lambda=str). Overridden per subclass."""
+        return raw
+
+    def _extra_to_dict(self) -> dict:
+        """Provider-specific fields, serialized between ``instance_id`` and the shared tail."""
+        return {}
+
+    @staticmethod
+    def _extra_from_dict(d: dict) -> dict:
+        """Provider-specific fields, parsed from a persisted handle dict (kwargs for the constructor)."""
+        return {}
+
+    def to_dict(self) -> dict:
+        return {
+            "provider": self.provider,
+            "instance_id": self.instance_id,
+            **self._extra_to_dict(),
+            "gpu": self.gpu,
+            "hourly_usd": self.hourly_usd,
+            "attempt": self.attempt,
+            "started_ts": self.started_ts,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> InstanceJobHandle:
+        try:
+            instance_id = cls._coerce_instance_id(d["instance_id"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise ValueError(
+                f"corrupt {cls.provider} handle: missing/non-numeric instance_id "
+                f"({d.get('instance_id')!r}) in persisted handle {d!r}"
+            ) from exc
+        return cls(
+            instance_id=instance_id,
+            gpu=str(d.get("gpu") or ""),
+            hourly_usd=float(d.get("hourly_usd") or 0),
+            attempt=int(d.get("attempt") or 0),
+            started_ts=float(d.get("started_ts") or 0),
+            **cls._extra_from_dict(d),
+        )
+
+
 # Fixed container path the per-region cache is bind-mounted at (host mount differs per provider).
 CACHE_CONTAINER_MOUNT = "/weight-cache"
 CACHE_HF_HOME = f"{CACHE_CONTAINER_MOUNT}/hf-cache"
@@ -131,7 +205,7 @@ def build_payload(
     bits the instance can't infer (HF prefix for markers, wall cap, attempt, and the substrate
     ``arm`` that the bootstrap stamps as FLASH_ARM + the marker name)."""
     from flash.envs.registry import worker_pip_for_env
-    from flash.providers.runpod.train import (
+    from flash.providers._worker import (
         build_worker_env,
         chalk_extra_pip,
         strip_runpod_volume_env,

{freesolo_flash_dev-0.2.38 → freesolo_flash_dev-0.2.40}/flash/providers/_instance_bootstrap.py

@@ -57,6 +57,9 @@ def _code_prefix(payload: dict) -> str:
 
 
 def _code_dir(payload: dict) -> str:
+    raw = payload.get("code_prefix")
+    if not isinstance(raw, str) or not raw.strip():
+        return os.path.join(CODE_ROOT, "code")
     return os.path.join(CODE_ROOT, os.path.dirname(_code_prefix(payload)) or ".")