freesolo-flash-dev 1.2.127__tar.gz → 1.2.128__tar.gz

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/CHANGELOG.md

@@ -25,6 +25,16 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Changed
 
+- A Flash run is one seed and may execute several attempts. The provider seed argument threaded
+  beside `spec.seed` is gone, so the spec is the only seed channel and no caller can supply a
+  second one to disagree with it; `Provider.submit_run`/`poll` are now `submit_attempt`/
+  `poll_attempt`; and `runner/supervise/seed_submission.py` is `attempt_supervision.py`. Instance
+  labels drop the redundant seed field (`<run>-s<seed>-a<n>` becomes `<run>-a<n>`), and every
+  RunPod endpoint carries an explicit `-a<n>` ordinal including attempt zero, replacing the `r<n>`
+  retry suffix that made a retry ordinal an identity and left the base name ambiguous between a run
+  and its first attempt. Names that exceed the provider budget now raise rather than truncate,
+  since a clipped ordinal is one two attempts of a run can collide on.
+
 - Run-backed adapters now use only explicit permanent checkpoint identities, `<run_id>/final` and
   `<run_id>/step-N`. Bare run aliases, composite source-revision identities, activation, implicit
   checkpoint fallback, and public artifact provenance have been removed across hosted, managed, and

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.127
+Version: 1.2.128
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/cli/commands/env/ops/imports.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 from flash.cli.commands.env.ops.push import (
     _ENV_EVALUATIONS_SIDECAR,
+    _decode_env_entrypoint_source,
     _ignore_env_push_path,
     _raise_walk_error,
 )
@@ -149,6 +150,18 @@ def _imported_module_names(tree, *, relative_names_are_siblings: bool = True) ->
     return names
 
 
+def _read_env_module_source(path: Path) -> str:
+    """Read one packaged Python file under the interpreter's own encoding contract.
+
+    the publish path decodes the entrypoint with `tokenize.detect_encoding`, so a file carrying a
+    non-utf-8 encoding cookie is publishable. reading it as utf-8 here raises instead, and every
+    caller below treats that as "no imports" -- so the sibling helpers never enter the archive and
+    `env push` exits 0 with an environment that only fails once a gpu has been rented. one decoder
+    for both paths is what keeps "publishable" and "walked for imports" the same set of files.
+    """
+    return _decode_env_entrypoint_source(path.read_bytes())[0]
+
+
 def _helper_imports(helper: Path, *, env_root: Path) -> set[str]:
     """Return imports needed by a packaged helper.
 
@@ -160,7 +173,7 @@ def _helper_imports(helper: Path, *, env_root: Path) -> set[str]:
         return set()
     try:
         return _imported_module_names(
-            ast.parse(helper.read_text(encoding="utf-8"), filename=str(helper)),
+            ast.parse(_read_env_module_source(helper), filename=str(helper)),
             relative_names_are_siblings=helper.parent == env_root,
         )
     except (OSError, SyntaxError, UnicodeDecodeError):
@@ -295,7 +308,7 @@ def _iter_env_sidecar_files(
         # GPU has been rented and the worker imports the module. parsed with the same reader as
         # every other module here, so a lazily imported helper (inside load_environment) counts.
         try:
-            entry_tree = ast.parse(entrypoint.read_text(encoding="utf-8"), filename=str(entrypoint))
+            entry_tree = ast.parse(_read_env_module_source(entrypoint), filename=str(entrypoint))
         except (OSError, SyntaxError, UnicodeDecodeError):
             # the entrypoint is validated for syntax on its own path, and a push that cannot read
             # it fails there with a better message. an unreadable file here just means no closure.
@@ -325,7 +338,7 @@ def _iter_env_sidecar_files(
                 yielded.add(sidecar)
                 yield sidecar, sidecar.relative_to(env_root)
             try:
-                tree = ast.parse(sidecar.read_text(encoding="utf-8"), filename=str(sidecar))
+                tree = ast.parse(_read_env_module_source(sidecar), filename=str(sidecar))
             except SyntaxError as exc:
                 raise ValueError(
                     f"{sidecar}: invalid evaluation sidecar syntax: {exc.msg}"

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/cli/commands/env/ops/push.py

@@ -382,15 +382,40 @@ def _env_name_error(raw: str) -> str:
     return f"env name invalid: {text!r} has no usable characters"
 
 
+def _validate_env_entrypoint_source(source: bytes) -> None:
+    """Reject exact Python bytes that cannot execute as an environment module."""
+    compile(source, _ENV_ENTRYPOINT, "exec")
+
+
+def _decode_env_entrypoint_source(source: bytes) -> tuple[str, str]:
+    """Decode Python bytes with the interpreter's encoding-cookie and BOM contract."""
+    import io
+    import tokenize
+
+    encoding, _ = tokenize.detect_encoding(io.BytesIO(source).readline)
+    return source.decode(encoding), encoding
+
+
+def _python_preamble_lines(lines: list[str]) -> int:
+    """Return the shebang and encoding-cookie lines that must remain first."""
+    if not lines:
+        return 0
+    encoding_cookie = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*[-\w.]+", re.ASCII)
+    blank_or_comment = re.compile(r"^[ \t\f]*(?:#|[\r\n]|$)", re.ASCII)
+    if encoding_cookie.match(lines[0]):
+        return 1
+    if len(lines) > 1 and blank_or_comment.match(lines[0]) and encoding_cookie.match(lines[1]):
+        return 2
+    return 1 if lines[0].startswith("#!") else 0
+
+
 def _with_syspath_bootstrap(env_source: str) -> str:
-    """Prepend a sys.path bootstrap so a published env can resolve shipped sibling helpers."""
+    """Insert the sys.path bootstrap after the Python module preamble."""
     import ast
 
-    try:
-        tree = ast.parse(env_source)
-    except SyntaxError:
-        return _ENV_SYSPATH_BOOTSTRAP + env_source
-    insert_after = 0
+    tree = ast.parse(env_source, filename=_ENV_ENTRYPOINT)
+    lines = env_source.splitlines(keepends=True)
+    insert_after = _python_preamble_lines(lines)
     body = tree.body
     i = 0
     if (
@@ -399,13 +424,27 @@ def _with_syspath_bootstrap(env_source: str) -> str:
         and isinstance(getattr(body[0], "value", None), ast.Constant)
         and isinstance(body[0].value.value, str)
     ):
-        insert_after = body[0].end_lineno or 0
+        insert_after = max(insert_after, body[0].end_lineno or 0)
         i = 1
     while i < len(body) and isinstance(body[i], ast.ImportFrom) and body[i].module == "__future__":
-        insert_after = body[i].end_lineno or insert_after
+        insert_after = max(insert_after, body[i].end_lineno or 0)
         i += 1
-    lines = env_source.splitlines(keepends=True)
-    return "".join(lines[:insert_after]) + _ENV_SYSPATH_BOOTSTRAP + "".join(lines[insert_after:])
+    prefix = "".join(lines[:insert_after])
+    if prefix and not prefix.endswith(("\n", "\r")):
+        prefix += "\n"
+    published_source = prefix + _ENV_SYSPATH_BOOTSTRAP + "".join(lines[insert_after:])
+    if not env_source.endswith(("\n", "\r")):
+        published_source = published_source.removesuffix("\n")
+    return published_source
+
+
+def _prepare_env_entrypoint_source(env_source: bytes) -> bytes:
+    """Validate the authored bytes and return the exact validated archive bytes."""
+    _validate_env_entrypoint_source(env_source)
+    decoded_source, encoding = _decode_env_entrypoint_source(env_source)
+    published_source = _with_syspath_bootstrap(decoded_source).encode(encoding)
+    _validate_env_entrypoint_source(published_source)
+    return published_source
 
 
 def _raise_walk_error(error: OSError) -> None:
@@ -674,6 +713,18 @@ def cmd_env_push(args) -> int:
     except ValueError as exc:
         return _err(str(exc))
 
+    try:
+        published_source = _prepare_env_entrypoint_source(entrypoint.read_bytes())
+    except (SyntaxError, UnicodeError, ValueError) as exc:
+        location = (
+            f"line {exc.lineno}" if getattr(exc, "lineno", None) is not None else "unknown line"
+        )
+        return _err(
+            f"cannot publish {src}: environment entrypoint has invalid Python syntax ({location})"
+        )
+    except OSError:
+        return _err(f"cannot publish {src}: environment entrypoint could not be read")
+
     try:
         _check_env_push_limits(
             env_root,
@@ -686,8 +737,7 @@ def cmd_env_push(args) -> int:
 
     with tempfile.TemporaryDirectory(prefix="flash-env-push-") as tmp:
         pkg = Path(tmp)
-        module_source = entrypoint.read_text()
-        (pkg / _ENV_ENTRYPOINT).write_text(_with_syspath_bootstrap(module_source))
+        (pkg / _ENV_ENTRYPOINT).write_bytes(published_source)
         _write_entrypoint_alias(pkg, entrypoint=entrypoint)
         _copy_env_sidecars(
             env_root,

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/cli/scaffold/TRAINING.md

@@ -255,7 +255,7 @@ model = "Qwen/Qwen3.5-9B"   # see `flash models list`
 project = "PROJECT_UUID"  # required UUID from `flash projects create`
 algorithm = "sft"           # "sft" (supervised), "grpo" (RL), or "opd" (on-policy distillation)
 # thinking = true           # opt-in reasoning mode, for models that support it
-# seed = 42                 # reproducible per-run seed; omitted defaults to 42
+# seed = 42                 # the run's one seed; omitted defaults to 42
 
 [environment]
 id = "your-org/your-project/my-env"      # the id printed by `flash env push`
@@ -364,7 +364,9 @@ remove that failover, and cannot be combined with `providers`. An unknown name i
 time and the error lists the names your plane accepts. `[gpu] type` pins one exact active
 validated GPU class. Run artifacts are stored in a
 private environment-scoped repo with content-addressed Flash code snapshots. Set `seed` only at the
-top level. Compose or tweak configs without editing files: `--config extra.toml` (deep-merge) and
+top level: a run has exactly one seed, and if a host fails Flash executes another attempt of that
+same run rather than a new one, so every attempt trains on the seed you set here.
+Compose or tweak configs without editing files: `--config extra.toml` (deep-merge) and
 `--set key=value` (e.g. `--set train.epochs=3`). `--gpus N` is
 shorthand for `--set gpu.count=N`; both land in one override list, so repeats resolve left to
 right.

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/cli/ui/heartbeat.py

@@ -69,7 +69,7 @@ def live_attempt(obj: dict) -> int | None:
 def heartbeat_is_current_attempt(obj: dict, heartbeat: dict) -> bool:
     """False only when the heartbeat provably belongs to a superseded retry attempt.
 
-    Retries reuse the seed's heartbeat path, so a recovered run flips back to ``running`` for the
+    Every attempt of a run shares one heartbeat path, so a recovered run flips back to ``running`` for the
     replacement worker while ``last_heartbeat`` can still be the previous attempt's setup ping until
     the new worker publishes one. ``remote.attempt`` is the live attempt; ``last_heartbeat.attempt``
     is the one that produced the ping. When the live attempt is known, keep the reassurance only for a

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/core/spec.py

@@ -268,18 +268,6 @@ CONTROL_PLANE_OWNED_ENV_KEYS = frozenset(
 TRAINER_BACKEND = "verl"
 
 
-def require_matching_seed(spec: JobSpec, seed: Any) -> int:
-    """Require the retained provider seed argument to match the authoritative JobSpec seed."""
-    provided = parse_seed(seed)
-    canonical = parse_seed(spec.seed)
-    if provided != canonical:
-        raise ValueError(
-            f"provider seed {provided} does not match JobSpec.seed {canonical}; "
-            "use spec.seed as the provider seed"
-        )
-    return canonical
-
-
 def _model_revision(value: Any) -> str:
     """parse an optional exact model-repository revision."""
     if not isinstance(value, str):

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/cost/spec.py

@@ -134,7 +134,7 @@ def _rollout_batch_for_quote(spec) -> int:
 
 
 def spec_steps(spec) -> int:
-    """Per-seed optimizer steps implied by a train spec (mirrors the worker).
+    """Per-run optimizer steps implied by a train spec (mirrors the worker).
 
     sft reads the horizon its workload profile measured: the profile already resolved epochs,
     retained rows, realized batch, and ``max_steps`` against the exact tokenized dataset, so

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/engine/worker/entry/worker.py

@@ -17,6 +17,7 @@ import flash.engine.worker.runtime.kernel_warmup as kernel_warmup
 import flash.engine.worker.train.entry.backend_common as backend_common
 from flash._internal.diagnostics import sanitize_diagnostic
 from flash.core.spec import gpu_count_of
+from flash.engine.support.huggingface import hub_error_transience
 from flash.engine.worker.runtime import state
 from flash.envs.loading.staged import (
     StagedEnvironmentTransientError,
@@ -67,6 +68,7 @@ def _run_worker_mode() -> None:
     # Idempotency: check DONE before any env-mutating pip install (fla fast path).
     if state.HF_REPO:
         from huggingface_hub import hf_hub_download
+        from huggingface_hub.errors import RemoteEntryNotFoundError
 
         try:
             hf_hub_download(
@@ -76,8 +78,15 @@ def _run_worker_mode() -> None:
                 token=os.environ.get("HF_TOKEN"),
             )
             done = True
-        except Exception:
-            done = False
+        except Exception as exc:
+            if type(exc) is RemoteEntryNotFoundError:
+                done = False
+            elif hub_error_transience(exc) is True:
+                raise worker_perf.RetriableInfraError(
+                    f"DONE marker read failed transiently ({type(exc).__name__})"
+                ) from exc
+            else:
+                raise
         remaining = state._remaining_worker_wall_seconds()
         if remaining is not None and remaining <= 0:
             raise RuntimeError("worker run wall deadline exceeded")

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/providers/_lifecycle/bootstrapping/bootstrap.py

@@ -22,6 +22,7 @@ from email.utils import parsedate_to_datetime
 if __package__:
     from flash.providers._lifecycle.bootstrapping import console as _bootstrap_console
     from flash.providers._lifecycle.bootstrapping import pip as bootstrap_pip
+    from flash.providers._lifecycle.bootstrapping import processes as _bootstrap_processes
     from flash.providers._lifecycle.bootstrapping.secrets import (
         _payload_secrets,
         _read_console_tail,
@@ -34,6 +35,7 @@ else:
     import archive as _source_snapshot  # type: ignore[no-redef]
     import bootstrap_console as _bootstrap_console  # type: ignore[no-redef]
     import bootstrap_pip  # type: ignore[no-redef]
+    import bootstrap_processes as _bootstrap_processes  # type: ignore[no-redef]
     from bootstrap_secrets import (  # type: ignore[no-redef]
         _payload_secrets,
         _read_console_tail,
@@ -297,9 +299,26 @@ def _upload_cleanup_deadlines(deadline_at: float) -> tuple[float, float]:
 
 
 def _worker_execution_deadline(upload_deadline_at: float) -> float:
+    """When the worker must be done, leaving the console upload window intact behind it."""
     return upload_deadline_at - _CONSOLE_UPLOAD_STOP_TIMEOUT_S - _CONSOLE_UPLOAD_FINAL_TIMEOUT_S
 
 
+def _group_teardown_deadline(upload_deadline_at: float) -> float:
+    """How long killing the worker's process group may take once the wall-clock cap has fired.
+
+    Teardown begins at the worker deadline, so it cannot be bounded by that same instant: a group
+    supervised against its own start time gets a zero wait, SIGKILL is never given time to be
+    reaped, and an ordinary capped run is reported as a surviving group holding the gpu.
+
+    What it must not eat into is the final console tail, since that tail carries the survivor line
+    to the user -- hence stopping one full ``_CONSOLE_UPLOAD_FINAL_TIMEOUT_S`` short of the upload
+    deadline. The uploader stop that runs in between is capped by its own constant against the same
+    absolute deadline, so it is not squeezed by a teardown that runs long. And running long is only
+    ever the stuck case: a group that dies on SIGKILL is reaped in milliseconds.
+    """
+    return upload_deadline_at - _CONSOLE_UPLOAD_FINAL_TIMEOUT_S
+
+
 def _join_upload_process_before(process, deadline_at: float, max_wait_s: float) -> None:
     remaining = max(
         0.0,
@@ -604,6 +623,7 @@ def run_mode(payload: dict, env: dict, mode: str, deadline_ts: float) -> int:
     console = f"/tmp/console_{mode}.txt"
     upload_deadline_at, reaping_deadline_at = _upload_cleanup_deadlines(deadline_ts)
     worker_deadline_at = _worker_execution_deadline(upload_deadline_at)
+    teardown_deadline_at = _group_teardown_deadline(upload_deadline_at)
     # cap work from its actual start: the absolute deadline includes boot grace, whose unused
     # portion must not extend the declared wall-time budget.
     budget = payload.get("run_max_wall_seconds")
@@ -615,12 +635,15 @@ def run_mode(payload: dict, env: dict, mode: str, deadline_ts: float) -> int:
                 _finite_positive_number(time.time(), "current clock") + budget,
             )
     timed_out = False
+    survived_teardown = ""
 
     with open(console, "w", buffering=1) as cf:
         code_dir = _code_dir(payload)
         if worker_deadline_at - _finite_positive_number(time.time(), "current clock") <= 0:
             raise TimeoutError(f"worker mode '{mode}' exceeded the wall-clock cap")
-        proc = subprocess.Popen(
+        # own group: the worker fans out into torchrun/vllm children that hold cuda contexts, and
+        # signalling only the leader leaves them holding the gpu after the cap kills the run.
+        proc, worker_process_group_id = _bootstrap_processes.start_process_group(
             [sys.executable, "-m", "flash.engine.support.worker_entrypoint"],
             cwd=code_dir,
             env={
@@ -679,13 +702,18 @@ def run_mode(payload: dict, env: dict, mode: str, deadline_ts: float) -> int:
                 proc.wait(timeout=remaining)
             except subprocess.TimeoutExpired:
                 timed_out = True
-                proc.kill()
-                remaining = max(
-                    0.0,
-                    worker_deadline_at - _finite_positive_number(time.time(), "current clock"),
-                )
-                with contextlib.suppress(subprocess.TimeoutExpired):
-                    proc.wait(timeout=remaining)
+                # the whole group, not just the leader: a surviving child keeps the gpu busy for
+                # every later run on this box. a survivor is recorded rather than raised here, so
+                # the console tail below still uploads and explains why the run was killed.
+                try:
+                    _bootstrap_processes.terminate_process_group(
+                        proc,
+                        process_group_id=worker_process_group_id,
+                        deadline_at=teardown_deadline_at,
+                    )
+                except RuntimeError as exc:
+                    survived_teardown = str(exc)
+                    print(survived_teardown, flush=True)
 
             drain_timeout = max(
                 0.0,
@@ -711,6 +739,8 @@ def run_mode(payload: dict, env: dict, mode: str, deadline_ts: float) -> int:
         extra = ""
         if timed_out:
             extra = f"\n--- bootstrap: mode '{mode}' hit the wall-clock cap; killed ---\n"
+            if survived_teardown:
+                extra += f"--- bootstrap: {survived_teardown} ---\n"
         if upload_deadline_at <= _finite_positive_number(time.time(), "current clock"):
             print("final console upload skipped; uploader reaping reserve reached", flush=True)
         elif not _upload_console_tail_bounded(
@@ -728,6 +758,12 @@ def run_mode(payload: dict, env: dict, mode: str, deadline_ts: float) -> int:
             flush=True,
         )
     if timed_out:
+        if survived_teardown:
+            # the box is still occupied, so this is not a clean capped run: say so loudly enough
+            # that the failure is not filed as an ordinary timeout.
+            raise TimeoutError(
+                f"worker mode '{mode}' exceeded the wall-clock cap and {survived_teardown}"
+            )
         raise TimeoutError(f"worker mode '{mode}' exceeded the wall-clock cap")
     return proc.returncode
 

freesolo_flash_dev-1.2.128/flash/providers/_lifecycle/bootstrapping/processes.py

@@ -0,0 +1,110 @@
+"""Process-group teardown for the worker the bootstrap launches.
+
+Stdlib only -- this ships inside the instance capsule alongside ``bootstrap.py`` and runs on the
+rented host, where the ``flash`` package is not importable. The worker's own
+``flash.engine.worker.train.entry.backend_common.kill_process_group`` solves the same problem one
+layer down, but the bootstrap cannot import it, so the group teardown it needs lives here.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import signal
+import subprocess
+import time
+
+_TERM_GRACE_S = 10.0
+_KILL_GRACE_S = 5.0
+_POLL_S = 0.1
+
+
+def _bounded_graces(
+    term_grace_s: float, kill_grace_s: float, deadline_at: float | None
+) -> tuple[float, float]:
+    """Split the time actually left before ``deadline_at`` across the term and kill waits.
+
+    The term wait yields first because the kill is the escalation that actually frees the gpu: with
+    a tight budget it is better to signal SIGKILL early and spend what remains waiting for it than
+    to burn the whole window on a SIGTERM the group is ignoring.
+    """
+    if deadline_at is None:
+        return max(0.0, term_grace_s), max(0.0, kill_grace_s)
+    remaining = max(0.0, deadline_at - time.time())
+    kill_wait = min(max(0.0, kill_grace_s), remaining)
+    term_wait = min(max(0.0, term_grace_s), remaining - kill_wait)
+    return term_wait, kill_wait
+
+
+def _group_exists(process_group_id: int) -> bool:
+    """Report whether any process still belongs to the group."""
+    try:
+        os.killpg(process_group_id, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        # visible but not ours to signal, so treat it as alive rather than silently done.
+        return True
+    return True
+
+
+def _group_exists_after_reaping(process: subprocess.Popen, process_group_id: int) -> bool:
+    """Reap the leader before probing, so a zombie leader does not read as a live group."""
+    process.poll()
+    return _group_exists(process_group_id)
+
+
+def _wait_for_group_exit(
+    process: subprocess.Popen, process_group_id: int, timeout_s: float
+) -> bool:
+    deadline = time.monotonic() + max(0.0, timeout_s)
+    while time.monotonic() < deadline:
+        if not _group_exists_after_reaping(process, process_group_id):
+            return True
+        time.sleep(_POLL_S)
+    return not _group_exists_after_reaping(process, process_group_id)
+
+
+def terminate_process_group(
+    process: subprocess.Popen,
+    *,
+    process_group_id: int,
+    term_grace_s: float = _TERM_GRACE_S,
+    kill_grace_s: float = _KILL_GRACE_S,
+    deadline_at: float | None = None,
+) -> None:
+    """SIGTERM the group, wait, escalate to SIGKILL, and verify the group is gone.
+
+    Signalling the group rather than the leader pid is the point: a training worker fans out into
+    torchrun and vllm children that hold cuda contexts, and one survivor strands a paid gpu for
+    every later run on that box.
+
+    ``deadline_at`` caps the whole supervision against the wall clock. Without it a group that
+    ignores SIGTERM costs the full term-then-kill grace, which is time the caller has already
+    budgeted for what comes after teardown -- the final console upload that carries the survivor
+    line. The kill still goes out when the term grace is spent, so the escalation is never skipped,
+    only the waiting is bounded.
+    """
+    term_wait, kill_wait = _bounded_graces(term_grace_s, kill_grace_s, deadline_at)
+    with contextlib.suppress(ProcessLookupError, PermissionError):
+        os.killpg(process_group_id, signal.SIGTERM)
+    if not _wait_for_group_exit(process, process_group_id, term_wait):
+        with contextlib.suppress(ProcessLookupError, PermissionError):
+            os.killpg(process_group_id, signal.SIGKILL)
+        _wait_for_group_exit(process, process_group_id, kill_wait)
+    with contextlib.suppress(subprocess.TimeoutExpired):
+        process.wait(timeout=0)
+    if _group_exists(process_group_id):
+        raise RuntimeError(f"process group {process_group_id} survived term and kill supervision")
+
+
+def start_process_group(args, **kwargs) -> tuple[subprocess.Popen, int]:
+    """Start one process as its own group leader and return it with its group id.
+
+    The group id is captured at launch because ``os.getpgid`` stops working once the leader is
+    reaped, which is exactly when the surviving descendants still need addressing.
+    """
+    if "start_new_session" in kwargs:
+        raise ValueError("start_new_session is owned by process-group supervision")
+    process = subprocess.Popen(args, start_new_session=True, **kwargs)
+    return process, process.pid

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.128}/flash/providers/_lifecycle/instances/instance.py

@@ -55,29 +55,28 @@ def run_label_prefix(run_id: str) -> str:
 
 
 def label_matches_run(label: str, prefix: str) -> bool:
-    """True iff ``label`` belongs to the run whose prefix is ``prefix`` — an EXACT match, or the prefix
-    followed by the ``-s`` seed boundary. Boundary-anchored so ``flash-100`` never matches
-    ``flash-1000-...`` (or vice versa). The single label-ownership test every instance provider's sweep
-    and run-scoped teardown shares."""
-    return label == prefix or label.startswith(prefix + "-s")
-
-
-def instance_label(run_id: str, seed: int, attempt: int) -> str:
-    """Instance name: run-derived so ``sweep_orphans`` can tell ours from anything else on the
-    account, and bounded (via ``run_label_prefix``) so the provider never truncates it."""
-    try:
-        seed_i = int(seed)
-    except (TypeError, ValueError):
-        seed_i = 0
+    """True iff ``label`` belongs to the run whose prefix is ``prefix`` -- an EXACT match, or the
+    prefix followed by the ``-a`` attempt boundary. Boundary-anchored so ``flash-100`` never matches
+    ``flash-1000-...`` (or vice versa). The single label-ownership test every instance provider's
+    sweep and run-scoped teardown shares."""
+    return label == prefix or label.startswith(prefix + "-a")
+
+
+def instance_label(run_id: str, attempt: int) -> str:
+    """Instance name: run plus the attempt executing on this host.
+
+    Run-derived so ``sweep_orphans`` can tell ours from anything else on the account, and bounded
+    (via ``run_label_prefix``) so the provider never truncates it. The seed is not part of the name:
+    a run has exactly one, so it distinguished nothing here while competing with the attempt for the
+    digit budget, and a truncated attempt is one two attempts of a run can collide on.
+    """
     attempt_i = _attempt_int(attempt)
     if attempt_i is None:
         raise ValueError("instance attempt identity is invalid")
-    seed_s, attempt_s = str(seed_i), str(attempt_i)
-    # Bound the whole suffix to _SUFFIX_BUDGET: split the digit budget between attempt and seed.
-    digit_budget = _SUFFIX_BUDGET - len("-s-a")
-    attempt_s = attempt_s[: max(1, min(len(attempt_s), max(1, digit_budget - 1)))]
-    seed_s = seed_s[: max(0, digit_budget - len(attempt_s))]
-    return f"{run_label_prefix(run_id)}-s{seed_s}-a{attempt_s}"
+    attempt_s = str(attempt_i)
+    if len(attempt_s) > _SUFFIX_BUDGET - len("-a"):
+        raise ValueError("instance attempt identity exceeds the provider name budget")
+    return f"{run_label_prefix(run_id)}-a{attempt_s}"
 
 
 @dataclass
@@ -183,7 +182,6 @@ fi
 
 def build_payload(
     spec,
-    seed: int,
     attempt: int,
     *,
     arm: str,
@@ -197,7 +195,6 @@ def build_payload(
     """The bootstrap's input — field-compatible with the RunPod ``_train_body`` payload, plus the
     bits the instance can't infer (HF prefix for markers, wall cap, attempt, and the substrate
     ``arm`` that the bootstrap stamps as FLASH_ARM + the marker name)."""
-    from flash.core.spec import require_matching_seed
     from flash.envs.loading.base import worker_pip_with_extras
     from flash.providers._lifecycle.net.worker import (
         build_worker_env,
@@ -205,12 +202,10 @@ def build_payload(
     )
     from flash.snapshot.archive import parse_descriptor
 
-    canonical_seed = require_matching_seed(spec, seed)
     # strip the runpod-only volume redirect; point base-model prefetch at this provider's cache unless the user overrode it.
     env = strip_runpod_volume_env(
         build_worker_env(
             spec,
-            canonical_seed,
             runtime_secrets=runtime_secrets,
         )
     )