freesolo-flash-dev 1.4.14__tar.gz → 1.4.16__tar.gz

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/.env.example

@@ -57,7 +57,7 @@ FREESOLO_INTERNAL_KEY=
 # Run without a Freesolo backend. Off by default. With it set, the plane stops calling
 # out for project/environment/billing validation and trusts FREESOLO_INTERNAL_KEY as a
 # single-tenant operator credential; external bearer tokens are REJECTED rather than
-# accepted unverified. Single tenant - see SELF_HOSTING.md before enabling.
+# accepted unverified. Single tenant: all authenticated clients share the same resources.
 # FLASH_STANDALONE=1
 
 # --- optional: deployment topology ---

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/.github/CODEOWNERS

@@ -21,6 +21,5 @@
 /pyproject.toml             @DavidBShan
 /uv.lock                    @DavidBShan
 
-# Security policy and the operator-facing credential template.
-/SECURITY.md                @DavidBShan
+# Operator-facing credential template.
 /.env.example               @DavidBShan

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -6,7 +6,7 @@ body:
     attributes:
       value: |
         Do not paste API keys, tokens, or credentials. Redact them before posting.
-        For security vulnerabilities, use private reporting instead (see SECURITY.md).
+        For security vulnerabilities, use private vulnerability reporting on GitHub instead.
 
   - type: textarea
     id: what-happened

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/.github/workflows/publish-serving-image.yml

@@ -128,7 +128,7 @@ jobs:
             publish=true
           elif ! changed="$(git diff --name-only "$last_published_sha" "$CURRENT_SHA" 2>/dev/null)"; then
             publish=true
-          elif grep -qE "^(flash/|docker/patch_vllm_moe_lora\.py$|Dockerfile\.serve$|\.dockerignore$|serve_launch\.py$|pyproject\.toml$|uv\.lock$|README\.md$|LICENSE$|NOTICE$|\.github/workflows/publish-serving-image\.yml$)" <<< "$changed"; then
+          elif grep -qE "^(flash/|docker/patch_vllm_moe_lora\.py$|Dockerfile\.serve$|\.dockerignore$|serve_launch\.py$|pyproject\.toml$|uv\.lock$|LICENSE$|NOTICE$|\.github/workflows/publish-serving-image\.yml$)" <<< "$changed"; then
             publish=true
           fi
           echo "publish=$publish" >> "$GITHUB_OUTPUT"

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/Dockerfile.serve

@@ -24,7 +24,7 @@ RUN python -c 'import sys; assert sys.version_info[:2] == (3, 12)' \
         --require-hashes -r /tmp/uv-requirements.txt \
     && rm -f /tmp/uv-requirements.txt
 
-COPY pyproject.toml uv.lock README.md LICENSE NOTICE ./
+COPY pyproject.toml uv.lock LICENSE NOTICE ./
 COPY flash ./flash
 COPY docker/patch_vllm_moe_lora.py ./docker/patch_vllm_moe_lora.py
 COPY serve_launch.py ./serve_launch.py

freesolo_flash_dev-1.4.16/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.5
+Name: freesolo-flash-dev
+Version: 1.4.16
+Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
+Project-URL: Homepage, https://github.com/freesolo-co/flash
+Project-URL: Repository, https://github.com/freesolo-co/flash
+Author: Freesolo
+License-Expression: Apache-2.0
+License-File: LICENSE
+License-File: NOTICE
+Keywords: fine-tuning,freesolo,grpo,llm,lora,rl,sft
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: <3.13,>=3.11
+Provides-Extra: custom
+Requires-Dist: torch==2.10.0; extra == 'custom'
+Provides-Extra: dev
+Requires-Dist: cloudpickle>=3.0; extra == 'dev'
+Requires-Dist: cryptography>=46; extra == 'dev'
+Requires-Dist: datasets>=2.19; extra == 'dev'
+Requires-Dist: fastapi<1,>=0.136; extra == 'dev'
+Requires-Dist: freesolo>=0.4.2; extra == 'dev'
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: huggingface-hub>=1.2.0; extra == 'dev'
+Requires-Dist: jsonschema>=4.23; extra == 'dev'
+Requires-Dist: modal==1.6.2; extra == 'dev'
+Requires-Dist: mypy>=1.13.0; extra == 'dev'
+Requires-Dist: orjson>=3.10; extra == 'dev'
+Requires-Dist: packaging<27,>=24; extra == 'dev'
+Requires-Dist: pillow>=11.0.0; extra == 'dev'
+Requires-Dist: pydantic-settings>=2.7; extra == 'dev'
+Requires-Dist: pytest>=9.0.3; extra == 'dev'
+Requires-Dist: referencing>=0.28.4; extra == 'dev'
+Requires-Dist: regex>=2024.11.6; extra == 'dev'
+Requires-Dist: requests<3,>=2.32; extra == 'dev'
+Requires-Dist: ruff<0.18,>=0.6; extra == 'dev'
+Requires-Dist: tokenizers>=0.22; extra == 'dev'
+Requires-Dist: transformers<5.13,>=5.8; extra == 'dev'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'dev'
+Provides-Extra: gpu
+Requires-Dist: accelerate>=1.4; extra == 'gpu'
+Requires-Dist: azure-identity<2,>=1.19; extra == 'gpu'
+Requires-Dist: azure-storage-blob<13,>=12.24; extra == 'gpu'
+Requires-Dist: datasets>=2.19; extra == 'gpu'
+Requires-Dist: freesolo>=0.4.2; extra == 'gpu'
+Requires-Dist: httpx>=0.27; extra == 'gpu'
+Requires-Dist: huggingface-hub>=1.2.0; extra == 'gpu'
+Requires-Dist: peft>=0.19; extra == 'gpu'
+Requires-Dist: requests<3,>=2.32; extra == 'gpu'
+Requires-Dist: tokenizers>=0.22; extra == 'gpu'
+Requires-Dist: torch==2.10.0; extra == 'gpu'
+Requires-Dist: transformers<5.13,>=5.8; extra == 'gpu'
+Requires-Dist: uv>=0.5; extra == 'gpu'
+Requires-Dist: vllm==0.19.1; extra == 'gpu'
+Provides-Extra: serve-runtime
+Requires-Dist: azure-identity<2,>=1.19; extra == 'serve-runtime'
+Requires-Dist: azure-storage-blob<13,>=12.24; extra == 'serve-runtime'
+Requires-Dist: fastapi<0.137,>=0.136; extra == 'serve-runtime'
+Requires-Dist: huggingface-hub>=1.2.0; extra == 'serve-runtime'
+Requires-Dist: pillow>=11.0.0; extra == 'serve-runtime'
+Requires-Dist: safetensors<1,>=0.8; extra == 'serve-runtime'
+Requires-Dist: tokenizers>=0.22; extra == 'serve-runtime'
+Requires-Dist: transformers<5.13,>=5.8; extra == 'serve-runtime'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'serve-runtime'
+Requires-Dist: vllm==0.23.0; extra == 'serve-runtime'
+Provides-Extra: server
+Requires-Dist: azure-identity<2,>=1.19; extra == 'server'
+Requires-Dist: azure-storage-blob<13,>=12.24; extra == 'server'
+Requires-Dist: cryptography>=46; extra == 'server'
+Requires-Dist: datasets>=2.19; extra == 'server'
+Requires-Dist: fastapi<1,>=0.136; extra == 'server'
+Requires-Dist: freesolo>=0.4.2; extra == 'server'
+Requires-Dist: httpx>=0.27; extra == 'server'
+Requires-Dist: huggingface-hub>=1.2.0; extra == 'server'
+Requires-Dist: jinja2>=3.1.0; extra == 'server'
+Requires-Dist: jsonschema>=4.23; extra == 'server'
+Requires-Dist: modal==1.6.2; extra == 'server'
+Requires-Dist: orjson>=3.10; extra == 'server'
+Requires-Dist: pillow>=11.0.0; extra == 'server'
+Requires-Dist: pydantic-settings>=2.7; extra == 'server'
+Requires-Dist: referencing>=0.28.4; extra == 'server'
+Requires-Dist: regex>=2024.11.6; extra == 'server'
+Requires-Dist: requests<3,>=2.32; extra == 'server'
+Requires-Dist: transformers<5.13,>=5.8; extra == 'server'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'server'
+Provides-Extra: serving
+Requires-Dist: anyio<5,>=4.0; extra == 'serving'
+Requires-Dist: azure-identity<2,>=1.19; extra == 'serving'
+Requires-Dist: azure-storage-blob<13,>=12.24; extra == 'serving'
+Requires-Dist: fastapi<1,>=0.136; extra == 'serving'
+Requires-Dist: httpx>=0.27.0; extra == 'serving'
+Requires-Dist: modal==1.6.2; extra == 'serving'
+Requires-Dist: orjson>=3.10; extra == 'serving'
+Requires-Dist: orjson>=3.10.0; extra == 'serving'
+Requires-Dist: pillow>=11.0.0; extra == 'serving'
+Requires-Dist: pydantic-settings>=2.6.0; extra == 'serving'
+Requires-Dist: pydantic-settings>=2.7; extra == 'serving'
+Requires-Dist: pydantic>=2.7.0; extra == 'serving'
+Requires-Dist: python-dotenv>=1.0.0; extra == 'serving'
+Requires-Dist: starlette<2,>=0.46; extra == 'serving'

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/_internal/channel.py

@@ -22,8 +22,8 @@ _CLI_SCRIPT_NAMES = (
     else frozenset({"flash", "flash-cli"})
 )
 
-# The `-m` form of the same entry point, which README/CONTRIBUTING/SELF_HOSTING all name as the
-# way around the shadowing. Not channel-dependent: the dev build renames the console scripts but
+# The `-m` form of the same entry point works around shadowing.
+# Not channel-dependent: the dev build renames the console scripts but
 # ships the identical `flash` package, so this module path is its escape hatch too.
 _CLI_MODULE = "flash.cli"
 

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/cli/commands/env/testing/eval.py

@@ -436,8 +436,8 @@ def _require_accessible_project(project_id: object) -> str:
             f"not logged in — run `{CLI_NAME} login` with your freesolo API key (or set FREESOLO_API_KEY)"
         )
     # pass `api_url` so self-hosted planes get shape validation without sending
-    # `FREESOLO_INTERNAL_KEY` to api.freesolo.co (SELF_HOSTING.md,
-    # `_verifies_against_freesolo`). upload may still fail later, but credential leakage is worse.
+    # `FREESOLO_INTERNAL_KEY` to api.freesolo.co (see `_verifies_against_freesolo`).
+    # upload may still fail later, but credential leakage is worse.
     return resolve_project_id(project_id, api_key, api_url)
 
 

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/client/__init__.py

@@ -36,9 +36,8 @@ def resolve_project(
     only validated for SHAPE -- exactly what ``flash/server/domain/registry/projects.py`` does under
     ``standalone()`` when the same run is submitted. Without this the ownership lookup went to
     ``api.freesolo.co``, which has no relationship with the operator's key and answers 401, so
-    ``flash env setup`` died before writing a file on the very quickstart SELF_HOSTING.md
-    documents. A backend-less plane exposes no project routes at all, so there is nothing else to
-    ask.
+    ``flash env setup`` died before writing a file on a standalone plane. A backend-less plane
+    exposes no project routes at all, so there is nothing else to ask.
 
     ``api_url`` is passed in rather than re-read from config because the caller has already
     resolved it (and because ``FLASH_STANDALONE`` lives on the SERVER, so the client cannot see

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/client/http.py

@@ -131,8 +131,8 @@ def has_freesolo_backend(api_url: str) -> bool:
     send a self-hosted plane's operator key to it.
 
     The presence of ``FREESOLO_API_KEY`` deliberately does NOT count. It looks like a hosted
-    signal but cannot be one: ``SELF_HOSTING.md`` has self-hosters log in with the
-    plane-controlling ``FREESOLO_INTERNAL_KEY``, and ``cmd_login`` reads that env var as the login
+    signal but cannot be one: self-hosters log in with the plane-controlling
+    ``FREESOLO_INTERNAL_KEY``, and ``cmd_login`` reads that env var as the login
     key, so its value is as likely to be the operator key as a hosted account key. Nothing marks
     which, and guessing wrong ships the plane credential to Freesolo. A hosted account reached
     from a self-hosted plane needs ``FREESOLO_BASE_URL`` set explicitly.

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/core/catalog.py

@@ -382,7 +382,7 @@ MODELS: dict[str, ModelInfo] = {
             max_cpu_loras=16,
             max_lora_rank=128,
             max_model_len=262144,
-            max_num_seqs=8,
+            max_num_seqs=16,
             tensor_parallel_size=1,
             gpu_memory_utilization=0.90,
             image_limit=MAX_IMAGES_PER_CONVERSATION,
@@ -549,7 +549,7 @@ MODELS["Qwen/Qwen3.5-9B-Base"] = replace(
         max_cpu_loras=16,
         max_lora_rank=128,
         max_model_len=262144,
-        max_num_seqs=8,
+        max_num_seqs=32,
         tensor_parallel_size=1,
         gpu_memory_utilization=0.90,
         image_limit=MAX_IMAGES_PER_CONVERSATION,
@@ -571,8 +571,7 @@ def get_model(model_id: str) -> ModelInfo:
         allowed = ", ".join(MODELS)
         raise ValueError(
             f"unsupported model {model_id!r}; choose one of: {allowed} — or, to train another "
-            f"model, fork Flash and add a ModelInfo entry for it to flash/core/catalog.py "
-            f"(see SELF_HOSTING.md)"
+            "model, fork Flash and add a ModelInfo entry for it to flash/core/catalog.py"
         ) from exc
 
 

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/custom/controller.py

@@ -58,9 +58,28 @@ def submit_custom(api, payload, *, client_train_schema=None):
         ) from error
 
 
-def connect(api, run_id, *, timeout=1200):
-    deadline = time.monotonic() + timeout
-    while time.monotonic() < deadline:
+_ATTACH_RETRY_MAX_SECONDS = 30.0
+
+
+def _transient_attach_error(exc):
+    """Whether a failed attach lookup can be retried rather than ending the run.
+
+    425 is the documented "GPU still provisioning" answer. A 5xx or a transport failure (no
+    status) means the control plane could not answer this time -- for example a slow Modal
+    tunnel lookup or an overloaded plane -- not that the run is gone. Treating those as fatal made
+    run_custom cancel healthy runs. Every other 4xx (ended run, auth, ownership) stays fatal.
+    """
+    if isinstance(exc, ApiError):
+        status = exc.status or 0
+        return status == 425 or status >= 500
+    return isinstance(exc, ClientError)
+
+
+def connect(api, run_id, *, timeout=1200, sleep=time.sleep, clock=time.monotonic):
+    deadline = clock() + timeout
+    failures = 0
+    last = None
+    while clock() < deadline:
         try:
             connection = api._request("GET", f"/v1/runs/{run_id}/custom")
             try:
@@ -70,13 +89,26 @@ def connect(api, run_id, *, timeout=1200):
                 client._http.timeout = 300
                 client.run_id = run_id
                 return client
-            except ClientError:
-                pass  # tunnel can exist before model loading finishes.
-        except ApiError as exc:
-            if exc.status != 425:
+            except ClientError as exc:
+                last = exc  # tunnel can exist before model loading finishes.
+            delay = 2.0
+        except ClientError as exc:
+            if not _transient_attach_error(exc):
                 raise
-        time.sleep(2)
-    raise TimeoutError(f"custom run {run_id} did not become ready")
+            last = exc
+            if isinstance(exc, ApiError) and exc.status == 425:
+                delay = 2.0
+            else:
+                failures += 1
+                delay = min(_ATTACH_RETRY_MAX_SECONDS, 2.0 * 2 ** min(failures, 4))
+                if failures == 1:
+                    print(
+                        f"control plane could not attach {run_id} yet ({exc}); retrying",
+                        file=sys.stderr,
+                        flush=True,
+                    )
+        sleep(max(0.0, min(delay, deadline - clock())))
+    raise TimeoutError(f"custom run {run_id} did not become ready ({last})")
 
 
 def run_custom(args, spec, authored_train_keys):

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/custom/session.py

@@ -13,6 +13,41 @@ from flash._internal.diagnostics import sanitize_diagnostic
 from flash.custom.types import integer
 
 
+def _reset_cuda_peak():
+    """Start a fresh per-operation peak; best effort, a no-op without CUDA."""
+    try:
+        import torch
+
+        if torch.cuda.is_available():
+            torch.cuda.reset_peak_memory_stats()
+    except Exception:
+        pass
+
+
+def _cuda_memory():
+    """GPU memory at the end of an operation (GiB), or None without CUDA.
+
+    Logged with every operation event so an out-of-memory failure shows whether memory crept up
+    over earlier operations or spiked inside the failing one. Never raises.
+    """
+    try:
+        import torch
+
+        if not torch.cuda.is_available():
+            return None
+        gib = 1024**3
+        free, total = torch.cuda.mem_get_info()
+        return {
+            "allocated_gib": round(torch.cuda.memory_allocated() / gib, 2),
+            "reserved_gib": round(torch.cuda.memory_reserved() / gib, 2),
+            "peak_allocated_gib": round(torch.cuda.max_memory_allocated() / gib, 2),
+            "free_gib": round(free / gib, 2),
+            "total_gib": round(total / gib, 2),
+        }
+    except Exception:
+        return None
+
+
 class SessionConflict(ValueError):
     """The caller must inspect/reconcile session state before doing more work."""
 
@@ -61,6 +96,9 @@ class TrainingSession:
         }
         if result.get("error"):
             event["error"] = result["error"]
+        memory = _cuda_memory()
+        if memory is not None:
+            event["cuda_memory"] = memory
         self.events.append(event)
         # prompts, token ids and credentials never enter worker stdout.
         print(json.dumps({"custom_training": event}, allow_nan=False), flush=True)
@@ -116,6 +154,7 @@ class TrainingSession:
                     "step": self.step,
                 }
             start = time.monotonic()
+            _reset_cuda_peak()
             try:
                 output = self.backend.execute(operation, arguments)
                 # reject invalid numerical results before committing a success receipt.

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/providers/core/base.py

@@ -596,7 +596,9 @@ class JobHandle:
 class PollResult:
     ok: bool
     metrics: dict | None = None
-    # failure: job_failed, oom, job_preempted, no_capacity, stalled, or poll_error.
+    # failure: job_failed, oom, job_preempted, no_capacity, stalled, poll_error, or
+    # checkpoint_upload_failed (training finished; the final CPU checkpoint upload failed after
+    # its own retries, and the committed handoff is preserved).
     failure: str | None = None
     detail: str | None = None
 

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/providers/core/preflight.py

@@ -113,7 +113,7 @@ def require_operator_config() -> None:
         raise PreflightError(
             "the Flash control plane is missing required operator configuration:\n"
             + "\n".join(problems)
-            + "\n\nSet these on the control-plane host. See SELF_HOSTING.md."
+            + "\n\nSet these on the control-plane host."
         )
 
 

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/providers/modal/checkpoint_upload.py

@@ -59,8 +59,20 @@ def cleanup_run(run_id: str) -> None:
         api.call(remove, timeout=api.TERMINATION_TIMEOUT_SECONDS)
 
 
-def _uploader_tags(handle):
-    return {
+# Training is finished and the GPU released by the time the CPU uploader runs, and the handoff
+# Volume keeps the committed checkpoint, so a failed upload is retried on a fresh uploader instead
+# of failing the run (`cpu_handoff` re-validates and re-uploads idempotently). Try 0 keeps the
+# original name and tags so uploads launched by an older control plane still reconcile.
+UPLOAD_TRIES = 3
+_UPLOAD_RETRY_DELAYS_S = (15.0, 60.0)
+
+
+class CheckpointUploadError(api.ModalApiError):
+    """Every CPU uploader try failed; the committed handoff Volume is preserved."""
+
+
+def _uploader_tags(handle, try_index: int = 0):
+    tags = {
         "flash_owner": "flash-training",
         "flash_run": handle.run_id,
         "flash_attempt": str(handle.attempt),
@@ -68,28 +80,51 @@ def _uploader_tags(handle):
         "flash_role": "checkpoint-upload",
         "flash_scope": handle.scope,
     }
+    if try_index:
+        tags["flash_upload_try"] = str(try_index)
+    return tags
 
 
-def _uploader_name(handle):
-    return "upload-" + handle.instance_id
+def _uploader_name(handle, try_index: int = 0):
+    name = "upload-" + handle.instance_id
+    return f"{name}-r{try_index}" if try_index else name
 
 
-async def _find_owned_uploader(modal, client, handle):
+async def _find_owned_uploader(modal, client, handle, try_index: int = 0):
     try:
         sandbox = await modal.Sandbox.from_name.aio(
             api.APP_NAME,
-            _uploader_name(handle),
+            _uploader_name(handle, try_index),
             environment_name=api.environment(),
             client=client,
         )
     except modal.exception.NotFoundError:
         return None
-    if await sandbox.get_tags.aio() != _uploader_tags(handle):
+    if await sandbox.get_tags.aio() != _uploader_tags(handle, try_index):
         raise UnreconciledCreateError("CPU uploader name belongs to another attempt")
     return sandbox
 
 
-def finish(handle, spec, *, deadline_at):
+async def _launched_spec_sha256(modal, client, handle):
+    """Bind publication to the original launch, not a reconstructed polling spec."""
+    sandbox = await modal.Sandbox.from_id.aio(handle.instance_id, client=client)
+    tags = await sandbox.get_tags.aio()
+    expected = {
+        "flash_owner": "flash-training",
+        "flash_run": handle.run_id,
+        "flash_attempt": str(handle.attempt),
+        "flash_name": handle.name,
+        "flash_handoff": volume_name(handle.scope, handle.run_id, handle.attempt),
+    }
+    if any(tags.get(key) != value for key, value in expected.items()):
+        raise api.ModalApiError("CPU checkpoint parent launch identity mismatch")
+    digest = tags.get("flash_spec")
+    if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
+        raise api.ModalApiError("CPU checkpoint parent launch has no valid specification digest")
+    return digest
+
+
+def finish(handle, *, deadline_at):
     """The GPU must be stopped and its Volume committed before attaching a CPU reader."""
     from flash.providers._lifecycle.net.worker import worker_image_for_gpu
 
@@ -103,7 +138,6 @@ def finish(handle, spec, *, deadline_at):
     environment = {
         **storage_environment,
         "FLASH_JOB_SPEC_PATH": f"{MOUNT}/spec.json",
-        "FLASH_EXPECTED_SPEC_SHA256": hashlib.sha256(spec.to_json().encode()).hexdigest(),
         "HF_REPO": handle.hf_repo,
         "RUN_ID": handle.run_id,
         "PHASE": handle.phase,
@@ -113,9 +147,22 @@ def finish(handle, spec, *, deadline_at):
         "PYTHONPATH": f"{MOUNT}/source",
     }
 
-    async def upload(modal, client):
-        sandbox = await _find_owned_uploader(modal, client, handle)
+    def uploader(try_index: int):
+        async def upload(modal, client):
+            await _run_uploader(modal, client, try_index)
+
+        return upload
+
+    async def _run_uploader(modal, client, try_index: int):
+        # a finished sandbox from an earlier try (or an earlier control plane) is not relaunched:
+        # its exit code is the result for that try.
+        sandbox = await _find_owned_uploader(modal, client, handle, try_index)
         if sandbox is None:
+            # Recovery reduces gpu.max_wall_seconds on the poll spec. Hashing that spec
+            # rejects the untouched handoff from the original GPU launch on every retry.
+            environment["FLASH_EXPECTED_SPEC_SHA256"] = await _launched_spec_sha256(
+                modal, client, handle
+            )
             app = await modal.App.lookup.aio(
                 api.APP_NAME,
                 environment_name=api.environment(),
@@ -134,8 +181,8 @@ def finish(handle, spec, *, deadline_at):
                     "-m",
                     "flash.engine.worker.io.cpu_handoff",
                     app=app,
-                    name=_uploader_name(handle),
-                    tags=_uploader_tags(handle),
+                    name=_uploader_name(handle, try_index),
+                    tags=_uploader_tags(handle, try_index),
                     image=modal.Image.from_registry(worker_image_for_gpu(handle.gpu)).pip_install(
                         "azure-storage-blob>=12.24,<13", "azure-identity>=1.19,<2"
                     ),
@@ -147,19 +194,42 @@ def finish(handle, spec, *, deadline_at):
                     client=client,
                 )
             except Exception:
-                sandbox = await _find_owned_uploader(modal, client, handle)
+                sandbox = await _find_owned_uploader(modal, client, handle, try_index)
                 if sandbox is None:
                     raise UnreconciledCreateError(
                         "CPU upload launch could not be reconciled"
                     ) from None
         await sandbox.wait.aio()
-        if await sandbox.poll.aio() != 0:
-            raise api.ModalApiError("CPU checkpoint upload failed; prior durable save is preserved")
+        returncode = await sandbox.poll.aio()
+        if returncode != 0:
+            raise api.ModalApiError(
+                f"CPU checkpoint uploader {sandbox.object_id} exited with code {returncode}"
+            )
 
-    api.call(upload, deadline_at=deadline_at, timeout=1900)
+    tries = 0
+    for try_index in range(UPLOAD_TRIES):
+        tries = try_index + 1
+        try:
+            api.call(uploader(try_index), deadline_at=deadline_at, timeout=1900)
+            break
+        except (api.ModalApiError, TimeoutError) as exc:
+            remaining = deadline_at - time.time()
+            if tries >= UPLOAD_TRIES or remaining <= 0:
+                raise CheckpointUploadError(
+                    f"CPU checkpoint upload failed after {tries} "
+                    f"{'try' if tries == 1 else 'tries'}; prior durable save is preserved; {exc}"
+                ) from exc
+            time.sleep(
+                min(
+                    _UPLOAD_RETRY_DELAYS_S[min(try_index, len(_UPLOAD_RETRY_DELAYS_S) - 1)],
+                    remaining,
+                )
+            )
     seconds = time.time() - started
     return {
         "cpu_upload_wall_seconds": seconds,
+        "cpu_upload_tries": tries,
+        # tries run one after another, so wall time already covers every uploader
         "cpu_upload_cost_upper_estimate_usd": seconds
         / 3600
         * (4 * CPU_HOURLY + 8 * MEMORY_GIB_HOURLY),
@@ -170,11 +240,12 @@ def cleanup(handle):
     """Cancel an owned CPU child before deleting its per-attempt storage."""
 
     async def remove(modal, client):
-        sandbox = await _find_owned_uploader(modal, client, handle)
-        if sandbox is not None:
-            await sandbox.terminate.aio(wait=True)
-            if await sandbox.poll.aio() is None:
-                raise api.ModalApiError("CPU uploader termination was not confirmed")
+        for try_index in range(UPLOAD_TRIES):
+            sandbox = await _find_owned_uploader(modal, client, handle, try_index)
+            if sandbox is not None:
+                await sandbox.terminate.aio(wait=True)
+                if await sandbox.poll.aio() is None:
+                    raise api.ModalApiError("CPU uploader termination was not confirmed")
         with contextlib.suppress(modal.exception.NotFoundError):
             await modal.Volume.objects.delete.aio(
                 volume_name(handle.scope, handle.run_id, handle.attempt),

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/providers/modal/jobs.py

@@ -146,11 +146,16 @@ def poll(handle, spec, *, log=None, deadline_at=None):
         from flash.providers.modal import checkpoint_upload
 
         try:
-            notes = checkpoint_upload.finish(h, spec, deadline_at=deadline_at)
-        except (api.ModalApiError, TimeoutError):
-            return PollResult(
-                False, failure="job_preempted", detail="Final CPU checkpoint publication failed"
+            notes = checkpoint_upload.finish(h, deadline_at=deadline_at)
+        except (api.ModalApiError, TimeoutError) as exc:
+            # training completed and the handoff Volume is preserved; this is an upload failure
+            # (finish already retried it), not a preemption -- report it as its own category.
+            detail = (
+                str(exc)
+                if isinstance(exc, checkpoint_upload.CheckpointUploadError)
+                else "Final CPU checkpoint publication failed"
             )
+            return PollResult(False, failure="checkpoint_upload_failed", detail=detail)
         result.metrics.setdefault("notes", {}).update(notes)
     return result
 

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/runner/results/checkpoints.py

@@ -102,7 +102,7 @@ def _checkpoints_from_files(spec: JobSpec, files: list[str]) -> list[dict]:
     return out
 
 
-def list_checkpoints(spec: JobSpec) -> list[dict]:
+def list_checkpoints(spec: JobSpec, *, strict: bool = False) -> list[dict]:
     """Deployable per-step adapter snapshots for ``spec``, ascending by step.
 
     A step is included only if its adapter folder carries BOTH ``adapter_config.json`` AND a
@@ -110,8 +110,12 @@ def list_checkpoints(spec: JobSpec) -> list[dict]:
     entry: ``{"step", "adapter_prefix", "subfolder", "repo_id", "repo_type"}`` where
     ``adapter_prefix`` is the value to hand ``deploy_adapter`` to serve that exact step and
     ``subfolder`` is the full path of the adapter folder in the repo. Returns ``[]`` when the
-    run has no artifact namespace or no published snapshots (older runs, or none saved yet)."""
-    return _checkpoints_from_files(spec, _repo_files(spec, strict=False))
+    run has no artifact namespace or no published snapshots (older runs, or none saved yet).
+
+    ``strict=True`` raises ``CheckpointListingError`` when the artifact listing itself fails,
+    instead of reporting that failure as "no checkpoints". User-facing listing and deploy routes
+    use it so a transient store error is not shown as a run whose checkpoints are gone."""
+    return _checkpoints_from_files(spec, _repo_files(spec, strict=strict))
 
 
 def adapter_artifact_exists(

{freesolo_flash_dev-1.4.14 → freesolo_flash_dev-1.4.16}/flash/serve/contract/urls.py

@@ -124,7 +124,7 @@ def serving_base_url() -> str:
             "A standalone plane has no serving backend of its own, and using the hosted one would "
             "send FREESOLO_INTERNAL_KEY - the key that controls this plane - to a service you do "
             "not operate. Point FREESOLO_SERVING_URL at your own multi-LoRA deployment, or export "
-            "the adapter and serve it yourself (see SELF_HOSTING.md). Training does not require "
+            "the adapter and serve it yourself. Training does not require "
             "this."
         )
     return serving_control_url(configured or default_serving_url())