freesolo-flash-dev 1.4.16__tar.gz → 1.4.18__tar.gz

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.4.16
+Version: 1.4.18
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/custom/backend.py

@@ -50,6 +50,7 @@ class TorchBackend:
         return validate_operation(self, operation, arguments)
 
     def execute(self, operation, arguments):
+        _ensure_cuda_headroom()
         return getattr(self, operation)(**arguments)
 
     def _tensor(self, tokens):
@@ -271,6 +272,35 @@ class TorchBackend:
         return {}
 
 
+# cuBLAS handle/workspace creation and Triton kernels allocate outside PyTorch's caching
+# allocator, so they need memory the driver can still hand out.
+CUDA_HEADROOM_BYTES = 6 * 1024**3
+
+
+def _ensure_cuda_headroom(min_free_bytes=CUDA_HEADROOM_BYTES):
+    """Return PyTorch's unused cached blocks to the driver when free GPU memory runs low.
+
+    A session's operations see many sequence lengths, so cached blocks rarely fit the next
+    request and the caching allocator keeps reserving: a resumed Qwen3.5-9B DPO session held
+    ~78 GiB reserved with only ~19 GiB allocated. The first backward then needed memory outside
+    that cache (cuBLAS handle creation, Triton kernels) and failed with CUBLAS_STATUS_ALLOC_FAILED
+    / "Triton Error [CUDA]: out of memory" at <1 GiB free. Releasing the cache before an operation
+    restores that headroom; it changes no tensors. Best effort, a no-op without CUDA.
+    """
+    try:
+        import torch
+
+        if not torch.cuda.is_available():
+            return False
+        free, _ = torch.cuda.mem_get_info()
+        if free >= min_free_bytes:
+            return False
+        torch.cuda.empty_cache()
+        return True
+    except Exception:
+        return False
+
+
 def _enable_checkpointing(model):
     """Checkpoint catalog models without active dropout or stateful normalization.
 

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/custom/controller.py

@@ -10,7 +10,7 @@ import uuid
 from pathlib import Path
 
 from flash.client import client_from_config
-from flash.client.http import ApiError, ClientError
+from flash.client.http import ApiError, ClientError, RequestTimeoutError, ServiceUnreachableError
 from flash.client.specs import spec_payload
 from flash.custom.async_client import AsyncTrainingClient
 from flash.custom.client import TrainingClient
@@ -64,15 +64,16 @@ _ATTACH_RETRY_MAX_SECONDS = 30.0
 def _transient_attach_error(exc):
     """Whether a failed attach lookup can be retried rather than ending the run.
 
-    425 is the documented "GPU still provisioning" answer. A 5xx or a transport failure (no
-    status) means the control plane could not answer this time -- for example a slow Modal
+    425 is the documented "GPU still provisioning" answer. A 5xx or a typed transport failure
+    means the control plane could not answer this time -- for example a slow Modal
     tunnel lookup or an overloaded plane -- not that the run is gone. Treating those as fatal made
-    run_custom cancel healthy runs. Every other 4xx (ended run, auth, ownership) stays fatal.
+    run_custom cancel healthy runs. Every other 4xx (ended run, auth, ownership) and untyped
+    ClientError (such as a malformed successful response) stays fatal.
     """
     if isinstance(exc, ApiError):
         status = exc.status or 0
         return status == 425 or status >= 500
-    return isinstance(exc, ClientError)
+    return isinstance(exc, (RequestTimeoutError, ServiceUnreachableError))
 
 
 def connect(api, run_id, *, timeout=1200, sleep=time.sleep, clock=time.monotonic):

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/engine/worker/train/core/child/glue.py

@@ -245,6 +245,19 @@ def _unique_glue_probe(messages: list[dict]) -> str:
             return probe
 
 
+# Chat templates such as Qwen3.5's raise "No user query found in messages" when a conversation
+# has no ordinary user turn; tool results are deliberately not one. Glue is the text after the
+# probe, which the turns before the probe do not change, so a tool-only reply is rendered after
+# this anchor to give the template the query it requires.
+_GLUE_ANCHOR = {"role": "user", "content": "Continue."}
+
+
+def _glue_conversation(probe: str, messages: list[dict]) -> list[dict]:
+    """the probe conversation: anchored only when the reply itself carries no user turn."""
+    head = [] if any(message.get("role") == "user" for message in messages) else [_GLUE_ANCHOR]
+    return [*head, {"role": "assistant", "content": probe}, *messages]
+
+
 def dedup_seam_terminator(response_ids: list[int], glue_ids: list[int]) -> list[int]:
     """keep the sampled terminator and drop the duplicate leading glue copy."""
     if response_ids and glue_ids and response_ids[-1] == glue_ids[0]:
@@ -269,7 +282,7 @@ class EnvGlueTokenizer:
             return list(cached)
         probe = _unique_glue_probe(messages)
         text = self.tokenizer.apply_chat_template(
-            _messages_for_chat_template([{"role": "assistant", "content": probe}, *messages]),
+            _messages_for_chat_template(_glue_conversation(probe, messages)),
             add_generation_prompt=True,
             tokenize=False,
             enable_thinking=self.thinking,
@@ -522,7 +535,7 @@ def _processor_glue_ids(
     structured = validate_structured_messages(messages, source="environment reply")
     probe = _unique_glue_probe(structured)
     rendered = processor.apply_chat_template(
-        [{"role": "assistant", "content": probe}, *structured],
+        _glue_conversation(probe, structured),
         tokenize=False,
         add_generation_prompt=True,
         **apply_chat_template_kwargs,

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/providers/_lifecycle/net/worker.py

@@ -135,10 +135,12 @@ def build_worker_env(
     # leaves rollout.enable_sleep_mode defaulted True, so the engine always builds a
     # CuMemAllocator -- which asserts outright on "expandable_segments:True"
     # (vllm/device_allocator/cumem.py:132, pytorch#147851). Both therefore take the
-    # non-expandable conf.
+    # non-expandable conf. SFT and custom runs never build a vLLM engine (custom samples with
+    # HF generate), so they take expandable segments: a custom session's varied sequence lengths
+    # otherwise fragment the caching allocator until cuBLAS/Triton allocations outside it fail.
     _alloc_conf = (
         "expandable_segments:True"
-        if str(getattr(spec, "algorithm", "")).lower() == "sft"
+        if str(getattr(spec, "algorithm", "")).lower() in ("sft", "custom")
         else "garbage_collection_threshold:0.8,max_split_size_mb:256"
     )
     env: dict[str, str] = {

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/runner/lifecycle/submit.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import logging
 import os
+from collections.abc import Callable
 from dataclasses import dataclass
 from functools import partial
 
@@ -256,11 +257,13 @@ def submit_job(
     prepared_job: PreparedJob | None = None,
     supervisor: Supervisor | None = None,
     queue_only: bool = False,
+    on_persisted: Callable[[str], None] | None = None,
 ) -> state.RunStatus:
     """Submit a prepared job, allocating resources only outside dry-run mode.
 
     ``queue_only`` persists the run as queued without launching it, for a control plane that does
-    not hold the plane lease; the holder's recovery scan launches it.
+    not hold the plane lease; the holder's recovery scan launches it. ``on_persisted``
+    durably acknowledges admission after status/secrets are saved, before dispatch.
     """
     if prepared_job is not None:
         prepared = prepared_job
@@ -273,6 +276,7 @@ def submit_job(
         )
     kwargs = {
         "dry_run": dry_run,
+        "on_persisted": on_persisted,
         "runtime_secrets": runtime_secrets,
         "billing_context": billing_context,
         "platform_context": platform_context,
@@ -298,6 +302,7 @@ def _submit_prepared_job(
     billing_context: dict | None,
     platform_context: dict | None,
     dispatch=None,
+    on_persisted: Callable[[str], None] | None = None,
 ) -> state.RunStatus:
     public_spec = prepared.public_spec
     worker_spec = prepared.worker_spec
@@ -361,6 +366,8 @@ def _submit_prepared_job(
         if not os.path.exists(state.runs_file_path(public_spec.run_id, ".json")):
             runtime_credentials.discard(public_spec.run_id)
         raise
+    if on_persisted is not None:
+        on_persisted(status.run_id)
     reporting._report_status(status)
     if dry_run:
         # A dry-run persists a state=dry_run record (retrievable, listable, and stageable for a

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/server/asgi/app.py

@@ -224,6 +224,7 @@ def create_app():
             meta,
             runs,
             serving,
+            submissions,
         )
     except ImportError as exc:
         raise RuntimeError(_SERVER_EXTRAS_HINT) from exc
@@ -309,6 +310,7 @@ def create_app():
     app.include_router(envs.router)
     app.include_router(env_files.router)
     app.include_router(runs.router)
+    app.include_router(submissions.router)
     app.include_router(custom.router)
     app.include_router(serving.router)
     app.include_router(adapter_download.router)

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/server/platform/db.py

@@ -29,6 +29,16 @@ CREATE TABLE IF NOT EXISTS runs (
   created_at REAL NOT NULL
 );
 CREATE INDEX IF NOT EXISTS runs_key_idx ON runs(key_id);
+CREATE TABLE IF NOT EXISTS run_submissions (
+  key_id INTEGER NOT NULL REFERENCES api_keys(id),
+  submission_id TEXT NOT NULL,
+  request_hash TEXT,
+  run_id TEXT NOT NULL UNIQUE,
+  accepted INTEGER NOT NULL DEFAULT 0,
+  cancelled INTEGER NOT NULL DEFAULT 0,
+  created_at REAL NOT NULL,
+  PRIMARY KEY (key_id, submission_id)
+);
 """
 
 
@@ -344,3 +354,45 @@ def all_runs() -> list[dict]:
     with _connect() as conn:
         rows = conn.execute("SELECT run_id, key_id, kind, created_at FROM runs").fetchall()
         return [dict(r) for r in rows]
+
+
+def reserve_submission(
+    key_id: int, submission_id: str, request_hash: str | None, run_id: str, *, cancelled=False
+) -> dict:
+    """Bind an owner-scoped logical submission to one run, durably before any launch."""
+    with _connect() as conn:
+        conn.execute(
+            "INSERT OR IGNORE INTO run_submissions "
+            "(key_id, submission_id, request_hash, run_id, created_at, cancelled) VALUES (?, ?, ?, ?, ?, ?)",
+            (key_id, submission_id, request_hash, run_id, time.time(), int(cancelled)),
+        )
+        if cancelled:
+            conn.execute(
+                "UPDATE run_submissions SET cancelled = 1 WHERE key_id = ? AND submission_id = ?",
+                (key_id, submission_id),
+            )
+        row = conn.execute(
+            "SELECT * FROM run_submissions WHERE key_id = ? AND submission_id = ?",
+            (key_id, submission_id),
+        ).fetchone()
+    return dict(row)
+
+
+def get_submission(key_id: int, submission_id: str) -> dict | None:
+    with _connect() as conn:
+        row = conn.execute(
+            "SELECT * FROM run_submissions WHERE key_id = ? AND submission_id = ?",
+            (key_id, submission_id),
+        ).fetchone()
+    return dict(row) if row else None
+
+
+def submission_for_run(run_id: str) -> dict | None:
+    with _connect() as conn:
+        row = conn.execute("SELECT * FROM run_submissions WHERE run_id = ?", (run_id,)).fetchone()
+    return dict(row) if row else None
+
+
+def accept_submission(run_id: str) -> None:
+    with _connect() as conn:
+        conn.execute("UPDATE run_submissions SET accepted = 1 WHERE run_id = ?", (run_id,))

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/server/platform/runtime.py

@@ -18,6 +18,7 @@ from flash.core.spec import JobSpec
 from flash.runner.lifecycle.state import adapter_prefix, runs_file_path
 from flash.runner.lifecycle.status import get_status
 from flash.server.platform import db
+from flash.server.platform.submission_admission import recovery_ready
 
 _log = logging.getLogger("flash.server")
 
@@ -263,6 +264,8 @@ def _start_handleless_resubmit(spec, expected_state: str) -> bool | None:
     )
     from flash.runner.supervise.lifecycle import _run_job_background
 
+    if not recovery_ready(spec.run_id):
+        return False
     try:
         status = get_status(spec.run_id)
         from flash.runner.lifecycle.runtime_credentials import restore
@@ -338,6 +341,8 @@ def _enqueue_resubmit(spec) -> bool:
     """Reserve execution capacity before acquiring any durable launch claim."""
     from flash.runner.lifecycle.status import source_snapshot_from_status
 
+    if not recovery_ready(spec.run_id):
+        return False
     try:
         status = get_status(spec.run_id)
         from flash.runner.lifecycle.runtime_credentials import restore
@@ -684,6 +689,12 @@ def _classify_recoverable_runs(
         _schedule_cleanup(status.run_id)
         if status.state not in _RECOVERABLE:
             continue
+        if not recovery_ready(status.run_id):
+            # A PUT replay/GET may briefly own the fence of an already-running job.
+            # Deferring recovery must not expose its live resources to orphan reaping.
+            if status.remote or status.cleanup_confirmed_remote:
+                active.add(status.run_id)
+            continue
         owner = current_supervisor()
         if any(owner.contains((kind, status.run_id)) for kind in ("run", "retry", "attach")):
             if status.remote or status.cleanup_confirmed_remote:

freesolo_flash_dev-1.4.18/flash/server/platform/submission_admission.py

@@ -0,0 +1,62 @@
+"""Shared admission fence for HTTP submissions and restart recovery (no server extras)."""
+
+from __future__ import annotations
+
+import fcntl
+import os
+from contextlib import contextmanager
+
+from flash.runner.lifecycle import state
+from flash.server.platform import db
+
+
+class SubmissionBusy(RuntimeError):
+    """Another process is still admitting or cancelling this submission."""
+
+
+@contextmanager
+def submission_lock(run_id: str):
+    # A slow owner cannot lose this fence to a timeout. The kernel releases on crash.
+    os.makedirs(state.RUNS_DIR, exist_ok=True)
+    fd = os.open(state.runs_file_path(run_id, ".submission.lock"), os.O_CREAT | os.O_RDWR, 0o600)
+    try:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise SubmissionBusy(run_id) from None
+        yield
+    finally:
+        os.close(fd)
+
+
+def recovery_ready(run_id: str) -> bool:
+    """Do not recover a status published by an unfinished submission.
+
+    Recovery may run on a different plane while PUT still owns the admission fence.
+    After a crash, the same fence lets recovery settle the durable cancellation intent
+    (using normal resource teardown) or finish acknowledgement before taking over.
+    Recheck at execution as well as classification: queued recovery tasks can be stale.
+    """
+    if db.submission_for_run(run_id) is None:
+        return True  # Legacy POST runs have no submission admission protocol.
+    try:
+        with submission_lock(run_id):
+            row = db.submission_for_run(run_id)
+            if row is None:
+                raise RuntimeError("durable submission disappeared during recovery")
+            from flash.runner.lifecycle.status import get_status
+
+            try:
+                get_status(run_id)
+            except FileNotFoundError:
+                return False
+            if row["cancelled"]:
+                from flash.runner.supervise.deploy import cancel_run
+
+                cancel_run(run_id)
+                return False
+            if not row["accepted"]:
+                db.accept_submission(run_id)
+            return True
+    except SubmissionBusy:
+        return False

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/server/routes/runs.py

@@ -354,15 +354,22 @@ def create_run(
     key: Annotated[dict, Depends(require_key)],
     authorization: Annotated[str | None, Header()] = None,
     x_freesolo_org_id: Annotated[str | None, Header()] = None,
+):
+    return _create_run(payload, request, key, authorization, x_freesolo_org_id)
+
+
+def _create_run(
+    payload, request, key, authorization, x_freesolo_org_id, *, run_id=None, on_persisted=None
 ):
     submission_started = False
+    reserved_identity = run_id is not None
     try:
         dry_run = _require_bool(payload, "dry_run", False)
         schema = _client_train_schema(payload)
         submitted = payload.get("spec")
         submitted_train = submitted.get("train") if isinstance(submitted, dict) else None
         try:
-            spec = _parse_spec(payload, run_id=new_run_id())
+            spec = _parse_spec(payload, run_id=run_id or new_run_id())
         except HTTPException as exc:
             detail = _schema_disagreement_detail(exc, schema, submitted_train)
             if detail is None:
@@ -430,8 +437,12 @@ def create_run(
                 estimate_usd=prepared.estimated_cost_usd,
                 org_id=affordability_org_id,
             )
+        owner = db.run_owner(run_id)
+        if owner is None:
+            db.record_run(run_id, key["id"])
+        elif owner != key["id"] or not reserved_identity:
+            raise HTTPException(409, "run ownership conflict")
         submission_started = True
-        db.record_run(run_id, key["id"])
         submit_kwargs = {
             "dry_run": dry_run,
             "background": True,
@@ -442,6 +453,8 @@ def create_run(
             # scan picks the queued run up (see flash.server.platform.plane_lease).
             "queue_only": getattr(request.app.state, "plane_standby", False),
         }
+        if on_persisted is not None:
+            submit_kwargs["on_persisted"] = on_persisted
         if runtime_secrets:
             submit_kwargs["runtime_secrets"] = runtime_secrets
         if billing_context:

freesolo_flash_dev-1.4.18/flash/server/routes/submissions.py

@@ -0,0 +1,151 @@
+"""Durable, owner-scoped training submissions, addressed by a caller's idempotency key.
+
+PUT /v1/run-submissions/{id} accepts the POST /v1/runs payload (paid runs only).
+Persist a random ID before sending and reuse that ID and payload on every retry.
+GET resolves the current job; DELETE cancels it or tombstones a not-yet-arrived intent.
+Both return submission_id, run_id and state; run_id is null before admission.
+A concurrent operation returns 409/Retry-After, payload conflicts 422, cancellations 410.
+
+Bindings live in the control plane's durable SQLite store, alongside run ownership,
+and never expire or disappear on job cancellation. IDs are scoped to the authenticated
+key, not globally. Runtime secret values may rotate and are never stored in the mapping.
+This contract depends on preserving the plane's existing shared durable state/lock volume;
+it cannot recover a lost database. Legacy POST /v1/runs remains non-idempotent.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from contextlib import contextmanager
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Header, HTTPException, Request
+
+from flash.runner.lifecycle import state, status
+from flash.server.platform import db
+from flash.server.platform.deps import require_key
+from flash.server.platform.submission_admission import SubmissionBusy, submission_lock
+from flash.server.routes import runs
+
+router = APIRouter()
+
+
+def _validate_id(submission_id: str) -> None:
+    if not re.fullmatch(r"[A-Za-z0-9_-]{16,128}", submission_id):
+        raise HTTPException(400, "submission ID must contain 16-128 letters, digits, '_' or '-'")
+
+
+@contextmanager
+def _submission_lock(run_id: str):
+    try:
+        with submission_lock(run_id):
+            yield
+    except SubmissionBusy:
+        raise HTTPException(
+            409, "submission is in progress; retry the same ID", headers={"Retry-After": "2"}
+        ) from None
+
+
+def _existing(row: dict) -> dict | None:
+    try:
+        result = status.get_status(row["run_id"]).to_dict()
+    except FileNotFoundError:
+        if row["accepted"]:
+            # A missing/corrupt status store is not permission to recreate paid work.
+            raise HTTPException(
+                503, "accepted submission status unavailable; retry the same ID"
+            ) from None
+        return None
+    db.accept_submission(row["run_id"])
+    owner = db.run_owner(row["run_id"])
+    if owner is None:
+        # Ownership may have been deliberately removed to prevent unsafe recovery.
+        raise HTTPException(503, "accepted submission ownership unavailable")
+    if owner != row["key_id"]:
+        raise HTTPException(409, "submission ownership conflict")
+    return {**result, "submission_id": row["submission_id"]}
+
+
+def _cancelled(row: dict) -> dict:
+    return {"submission_id": row["submission_id"], "state": "cancelled", "run_id": None}
+
+
+@router.put("/v1/run-submissions/{submission_id}")
+def submit(
+    submission_id: str,
+    payload: dict,
+    request: Request,
+    key: Annotated[dict, Depends(require_key)],
+    authorization: Annotated[str | None, Header()] = None,
+    x_freesolo_org_id: Annotated[str | None, Header()] = None,
+):
+    _validate_id(submission_id)
+    if payload.get("dry_run", False) is not False:
+        raise HTTPException(
+            400, "idempotent submissions are for paid runs; preview with POST /v1/runs"
+        )
+    # Credentials can rotate without changing the logical job. Never persist secret values
+    # or hashes of low-entropy secrets. The spec still binds the requested secret names.
+    identity = {name: value for name, value in payload.items() if name != "runtime_secrets"}
+    identity.setdefault("dry_run", False)
+    identity["org_id"] = key.get("org_id") or x_freesolo_org_id
+    digest = hashlib.sha256(
+        json.dumps(identity, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
+    ).hexdigest()
+    row = db.reserve_submission(key["id"], submission_id, digest, state.new_run_id())
+    with _submission_lock(row["run_id"]):
+        row = db.get_submission(key["id"], submission_id)
+        if row["request_hash"] not in (None, digest):
+            raise HTTPException(422, "submission ID is already bound to a different request")
+        if row["cancelled"]:
+            raise HTTPException(410, "submission was cancelled; it will never create another run")
+        if result := _existing(row):
+            return result
+
+        def accepted(run_id):
+            db.accept_submission(run_id)
+            if db.get_submission(key["id"], submission_id)["cancelled"]:
+                status._update(run_id, "cancelled")
+                raise HTTPException(410, "submission was cancelled before dispatch")
+
+        result = runs._create_run(
+            payload,
+            request,
+            key,
+            authorization,
+            x_freesolo_org_id,
+            run_id=row["run_id"],
+            on_persisted=accepted,
+        )
+        db.accept_submission(row["run_id"])
+        return {**result, "submission_id": submission_id}
+
+
+@router.get("/v1/run-submissions/{submission_id}")
+def lookup(submission_id: str, key: Annotated[dict, Depends(require_key)]):
+    _validate_id(submission_id)
+    row = db.get_submission(key["id"], submission_id)
+    if row is None:
+        raise HTTPException(404, "unknown submission ID")
+    with _submission_lock(row["run_id"]):
+        row = db.get_submission(key["id"], submission_id)
+        if result := _existing(row):
+            return result
+        if row["cancelled"]:
+            return _cancelled(row)
+        return {"submission_id": submission_id, "run_id": None, "state": "submission_pending"}
+
+
+@router.delete("/v1/run-submissions/{submission_id}")
+def cancel(submission_id: str, request: Request, key: Annotated[dict, Depends(require_key)]):
+    _validate_id(submission_id)
+    # A cancellation arriving before a delayed PUT leaves a permanent tombstone.
+    row = db.reserve_submission(key["id"], submission_id, None, state.new_run_id(), cancelled=True)
+    with _submission_lock(row["run_id"]):
+        row = db.get_submission(key["id"], submission_id)
+        if _existing(row) is None:
+            return _cancelled(row)
+        result = runs.cancel(row["run_id"], request, key)
+        return {**result, "submission_id": submission_id}

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/serving/src/engine/model_config.py

@@ -64,7 +64,8 @@ SERVING_MODELS: list[dict[str, Any]] = [
             "max_cpu_loras": 4,
             "max_lora_rank": 64,
             "max_model_len": 262144,
-            "max_num_seqs": 8,
+            # L4 sweep: 64 sequences improve short/4k throughput ~3x/~1.6x over eight.
+            "max_num_seqs": 64,
             "enforce_eager": False,
             "reasoning_parser": "qwen3",
         },
@@ -105,9 +106,9 @@ SERVING_MODELS: list[dict[str, Any]] = [
             "max_loras": 16,
             "max_lora_rank": 128,  # rank-128 / 16 hot LoRAs (cheap on the 9 GiB FP8 9B); native 256k context.
             "max_model_len": 262144,
-            # L40S throughput + rank-128/32k-prompt sweep qualified sixteen sequences.
-            # Thirty-two 32k prompts exceed this tier's measured ~780k-token KV capacity.
-            "max_num_seqs": 16,
+            # L40S: 64 improves short/4k throughput ~127%/~40% over sixteen and
+            # completes a full 64-request 32k-prompt rank-128 q_proj LoRA burst.
+            "max_num_seqs": 64,
             "enforce_eager": False,
             "reasoning_parser": "qwen3",
         },
@@ -117,8 +118,8 @@ SERVING_MODELS: list[dict[str, Any]] = [
         "image_input_limit": MAX_IMAGES_PER_CONVERSATION,
         "gpu": "H100",
         "engine": {
-            "serve_model_id": "Qwen/Qwen3.5-9B-Base",
-            "model_revision": "68c46c4b3498877f3ef123c856ecfde50c39f404",
+            # FP8 conversion of the pinned Base weights; never alias the Instruct checkpoint.
+            "model_revision": "b1a2cf5f5e7bb4ad786e206f834569be3a3d8aa9",
             "tokenizer_model": "Qwen/Qwen3.5-9B-Base",
             "tokenizer_revision": "68c46c4b3498877f3ef123c856ecfde50c39f404",
             "processor_revision": "68c46c4b3498877f3ef123c856ecfde50c39f404",
@@ -128,9 +129,8 @@ SERVING_MODELS: list[dict[str, Any]] = [
             "max_cpu_loras": 16,
             "max_lora_rank": 128,
             "max_model_len": 262144,
-            # Single-H100 sweep qualified thirty-two sequences, including rank-128
-            # q_proj LoRA at 32k prompts; benchmark_qwen_serving.py reproduces the check.
-            "max_num_seqs": 32,
+            # Keep Modal admission aligned with this per-container scheduler capacity.
+            "max_num_seqs": 64,
             "enforce_eager": False,
             "reasoning_parser": "qwen3",
         },
@@ -259,7 +259,7 @@ def gpu_for(base_model: str) -> str:
 def serve_model_for(base_model: str) -> str:
     """return the pre-quantized checkpoint an active hosted engine loads."""
     _config_for(base_model)
-    if base_model in {"Qwen/Qwen3.5-0.8B", "Qwen/Qwen3.5-4B", "Qwen/Qwen3.5-9B-Base"}:
+    if base_model in {"Qwen/Qwen3.5-0.8B", "Qwen/Qwen3.5-4B"}:
         return base_model
     if base_model == DEEPSEEK_MODEL_ID:
         return DEEPSEEK_MODEL_ID

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/flash/serving/src/engine/prequant_config.py

@@ -1,6 +1,7 @@
 """Prequantized checkpoint mapping for active hosted serving.
 
-The 9B loads a Freesolo-owned compressed-tensors FP8 checkpoint. The 27B loads its official FP8
+The 9B Base and Instruct load distinct Freesolo-owned compressed-tensors FP8 checkpoints.
+The 27B loads its official FP8
 checkpoint. The 35B MoE mapping remains its official FP8 default, although the validated engine
 override serves the base bf16 weights so full-expert LoRA and CUDA graphs coexist.
 """
@@ -10,6 +11,7 @@ from __future__ import annotations
 # freesolo-owned compressed-tensors checkpoints.
 OWNED_FP8_MODEL_REPOS: dict[str, str] = {
     "Qwen/Qwen3.5-9B": "Freesolo-Co/Qwen3.5-9B-FP8",
+    "Qwen/Qwen3.5-9B-Base": "Freesolo-Co/Qwen3.5-9B-Base-FP8",
 }
 
 # official qwen checkpoints active in hosted serving.

{freesolo_flash_dev-1.4.16 → freesolo_flash_dev-1.4.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.4.16"
+version = "1.4.18"
 description ="Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI"
 # The training and serving stack is validated on Python 3.11 and 3.12.
 requires-python = ">=3.11,<3.13"
@@ -210,7 +210,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.4.16"
+version = "1.4.18"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]