freesolo-flash-dev 1.2.117__tar.gz → 1.2.118__tar.gz

{freesolo_flash_dev-1.2.117 → freesolo_flash_dev-1.2.118}/CHANGELOG.md

@@ -13,6 +13,10 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Added
 
+- OpenAI chat serving now preserves mirror sampling controls across packaged, hosted, and managed
+  entry points: up to four indexed choices, signed seeds, frequency and presence penalties, and
+  OpenAI token logprobs. Buffered and raw SSE responses retain independent choice terminals and
+  aggregate request usage; the decoded text-only stream remains single-choice.
 - A `flash-cli` console script, the same entry point as `flash` under a name nothing else
   claims. The `server` and `dev` extras install `runpod-flash`, which declares its own `flash`
   script; whichever distribution is installed last wins, so on a control-plane host

{freesolo_flash_dev-1.2.117 → freesolo_flash_dev-1.2.118}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.117
+Version: 1.2.118
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.117 → freesolo_flash_dev-1.2.118}/docs/serving-contract.md

@@ -139,15 +139,39 @@ unactivated aliases return `404` or `503`.
 The registered grammar is the default for every call to that revision. A request may override it
 per call with `structured_outputs`, or with the OpenAI-standard `response_format` accepted at this
 endpoint only; `{}` (equivalently `response_format: {"type": "text"}`) means explicitly
-unconstrained for that call and does not change what is registered. Requests may also include
-`stop` and `chat_template_kwargs.enable_thinking`.
+unconstrained for that call and does not change what is registered. Requests may also include `stop` and `chat_template_kwargs.enable_thinking`.
+
+The packaged, hosted, and managed OpenAI entry points share one strict sampling grammar:
+
+- `n` is an exact integer from 1 through 4. `n > 1` requires `temperature > 0`; the server never
+  rewrites temperature or duplicates a greedy output.
+- `seed` is null or a signed 64-bit integer. Explicit `-1` is rejected because vLLM treats it as an
+  unset seed.
+- `frequency_penalty` and `presence_penalty` are finite numbers from -2 through 2.
+- `logprobs` is a strict boolean and `top_logprobs` is an exact integer from 0 through 20. A positive
+  `top_logprobs` requires `logprobs: true`.
+- Thinking-enabled adapters reject logprobs after the requested adapter or run has been authorized
+  and resolved. Tools, tool selection, and unrelated vLLM extensions remain unsupported.
+
+A buffered response contains one indexed `choices` entry per generated choice. Each choice keeps its
+own finish reason and, when requested, OpenAI token logprobs. Prompt and cached tokens count once per
+request; completion tokens are summed across all choices. Streaming may interleave choice indexes.
+Each choice has independent reasoning state and exactly one terminal entry, followed by one aggregate
+usage block when requested and one final `data: [DONE]`. Hosted serving durably settles the single
+request lifecycle before emitting successful terminals. Managed serving forwards the backend's raw
+SSE frames without flattening choices or rewriting provenance.
+
+The decoded convenience `chat_stream` iterator remains text-only and single-choice. It rejects
+`n != 1` and any logprob request before opening transport; use buffered JSON or raw SSE for those
+features.
+
 Every successful response reports the exact revision, checkpoint, and Hugging Face commit in:
 
 - body object `freesolo` with `adapter_revision`, `checkpoint`, and `hf_revision`;
 - headers `X-Freesolo-Adapter-Revision`, `X-Freesolo-Checkpoint`, and
   `X-Freesolo-HF-Revision`.
-  Streaming uses `text/event-stream` with `data:` JSON frames containing
-  `choices[0].delta.content`, followed by `data: [DONE]`. Preserve stop handling and provenance headers.
+  Streaming uses `text/event-stream` with indexed `data:` JSON choice frames followed by
+  `data: [DONE]`. Preserve stop handling and provenance headers.
   A cold adapter may return `503` with a top-level retryable `adapter_unavailable` /
   `adapter_loading` error envelope and a `Retry-After` header.
   The shared runtime supports bounded multimodal preparation. An image-capable profile declares

{freesolo_flash_dev-1.2.117 → freesolo_flash_dev-1.2.118}/flash/engine/worker/train/rl/launch/checkpoints.py

@@ -13,6 +13,7 @@ import time
 import flash.engine.worker.io.hf as _worker_hf
 import flash.engine.worker.runtime.state as _worker_state
 import flash.engine.worker.verl.checkpoints as verl_checkpoints
+from flash._internal.fileio import reject_duplicate_keys
 from flash.adapters.artifacts import MAX_ATTEMPT_ID, has_loadable_adapter_weights
 from flash.engine.support.verl_checkpoint import FsdpCheckpointInspection
 from flash.engine.support.verl_policy import FsdpGeneration
@@ -69,6 +70,15 @@ def _strict_attempt(value, *, field: str) -> int:
     return value
 
 
+class _DuplicateJsonKeyError(ValueError):
+    pass
+
+
+_REJECT_DUPLICATE_MANIFEST_KEYS = reject_duplicate_keys(
+    lambda key: _DuplicateJsonKeyError(f"duplicate key {key!r}")
+)
+
+
 def _read_resume_manifest(
     checkpoint_dir: str,
     *,
@@ -83,8 +93,8 @@ def _read_resume_manifest(
         )
     try:
         with open(path, encoding="utf-8") as handle:
-            manifest = json.load(handle)
-    except (OSError, json.JSONDecodeError) as error:
+            manifest = json.load(handle, object_pairs_hook=_REJECT_DUPLICATE_MANIFEST_KEYS)
+    except (OSError, UnicodeError, json.JSONDecodeError, _DuplicateJsonKeyError) as error:
         raise RuntimeError(
             f"GRPO resume checkpoint {checkpoint_step} has an unreadable {_RESUME_MANIFEST}"
         ) from error
@@ -510,43 +520,44 @@ def _restore_verl_resume(
     required_steps: tuple[int, ...] = (),
 ) -> int:
     """stage this run's current-contract native checkpoint and its internal adapter store."""
-    inspections: dict[str, FsdpCheckpointInspection] = {}
+    accepted: dict[str, tuple[tuple[int, int], ...]] = {}
+    rejected: dict[str, RuntimeError] = {}
 
-    def prefer(path: str) -> bool:
-        inspection = inspections.get(path)
-        if inspection is None:
-            inspection = verl_checkpoints.inspect_resume_checkpoint(
-                path,
-                world_size=world_size,
-                expected_fsdp_generation=expected_fsdp_generation,
-            )
-            inspections[path] = inspection
-        if not inspection.loadable:
+    def accept(path: str, _inspection: FsdpCheckpointInspection) -> bool:
+        if path in accepted:
+            return True
+        if path in rejected:
             return False
-        step_text = os.path.basename(path).removeprefix("checkpoint-")
-        if not step_text.isdigit() or int(step_text) <= 0:
+        name = os.path.basename(path)
+        if not name.startswith("checkpoint-") or not name[len("checkpoint-") :].isdigit():
+            rejected[path] = RuntimeError(f"invalid GRPO resume checkpoint path {path!r}")
+            return False
+        step = int(name[len("checkpoint-") :])
+        if step <= 0:
+            rejected[path] = RuntimeError(f"invalid GRPO resume checkpoint path {path!r}")
             return False
         try:
             referenced, _positive = _read_resume_manifest(
                 path,
-                checkpoint_step=int(step_text),
+                checkpoint_step=step,
                 required_steps=required_steps,
             )
             _internal_adapter_sources(path, referenced)
-        except RuntimeError:
+        except RuntimeError as error:
+            rejected[path] = error
             return False
+        accepted[path] = referenced
         return True
 
-    resume = _worker_hf.hf_resume_checkpoint(prefer=prefer)
+    resume, inspection = verl_checkpoints.select_verl_resume_checkpoint(
+        world_size=world_size,
+        expected_fsdp_generation=expected_fsdp_generation,
+        accept=accept,
+    )
     if not resume:
         return 0
-    inspection = inspections.get(resume)
     if inspection is None:
-        inspection = verl_checkpoints.inspect_resume_checkpoint(
-            resume,
-            world_size=world_size,
-            expected_fsdp_generation=expected_fsdp_generation,
-        )
+        raise RuntimeError("selected GRPO resume checkpoint is missing its inspection")
     if not inspection.loadable:
         return verl_checkpoints.stage_verl_resume(
             resume,
@@ -556,15 +567,12 @@ def _restore_verl_resume(
             expected_fsdp_generation=expected_fsdp_generation,
             inspection=inspection,
         )
-    name = os.path.basename(resume)
-    if not name.startswith("checkpoint-") or not name[len("checkpoint-") :].isdigit():
+    referenced = accepted.get(resume)
+    if referenced is None:
+        error = rejected.get(resume)
+        if error is not None:
+            raise error
         raise RuntimeError(f"invalid GRPO resume checkpoint path {resume!r}")
-    step = int(name[len("checkpoint-") :])
-    referenced, _positive = _read_resume_manifest(
-        resume,
-        checkpoint_step=step,
-        required_steps=required_steps,
-    )
     _stage_internal_adapters(resume, local_dir, referenced)
     return verl_checkpoints.stage_verl_resume(
         resume,

{freesolo_flash_dev-1.2.117 → freesolo_flash_dev-1.2.118}/flash/engine/worker/verl/checkpoints.py

@@ -16,7 +16,7 @@ import os
 import re
 import shutil
 import subprocess
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from typing import Any
 
 from flash.adapters.fused_experts import (
@@ -318,19 +318,13 @@ def inspect_resume_checkpoint(
     )
 
 
-def restore_verl_resume(
-    local_dir: str,
+def select_verl_resume_checkpoint(
     *,
     world_size: int,
     expected_fsdp_generation: FsdpGeneration,
-    job_label: str,
-) -> int:
-    """select, inspect once, and stage the newest loadable streamed checkpoint.
-
-    the artifact fetch prefers a lower loadable checkpoint over a higher incompatible one, so a
-    repeated discard cannot starve compatible state uploaded after the rejected checkpoint. the
-    cached inspection is passed into staging so selection and diagnostics share one verdict.
-    """
+    accept: Callable[[str, FsdpCheckpointInspection], bool] | None = None,
+) -> tuple[str | None, FsdpCheckpointInspection | None]:
+    """select the newest caller-accepted loadable checkpoint and return its cached inspection."""
     import flash.engine.worker.io.hf as worker_hf
 
     inspections: dict[str, FsdpCheckpointInspection] = {}
@@ -344,18 +338,41 @@ def restore_verl_resume(
                 expected_fsdp_generation=expected_fsdp_generation,
             )
             inspections[path] = inspection
-        return inspection.loadable
+        return inspection.loadable and (accept is None or accept(path, inspection))
 
     resume = worker_hf.hf_resume_checkpoint(prefer=prefer)
+    if not resume:
+        return None, None
+    inspection = inspections.get(resume)
+    if inspection is None:
+        prefer(resume)
+        inspection = inspections[resume]
+    return resume, inspection
+
+
+def restore_verl_resume(
+    local_dir: str,
+    *,
+    world_size: int,
+    expected_fsdp_generation: FsdpGeneration,
+    job_label: str,
+) -> int:
+    """select, inspect once, and stage the newest loadable streamed checkpoint."""
+    resume, inspection = select_verl_resume_checkpoint(
+        world_size=world_size,
+        expected_fsdp_generation=expected_fsdp_generation,
+    )
     if not resume:
         return 0
+    if inspection is None:
+        raise RuntimeError("selected verl resume checkpoint is missing its inspection")
     return stage_verl_resume(
         resume,
         local_dir,
         job_label=job_label,
         world_size=world_size,
         expected_fsdp_generation=expected_fsdp_generation,
-        inspection=inspections.get(resume),
+        inspection=inspection,
     )
 
 

freesolo_flash_dev-1.2.118/flash/serve/app/chat_stream.py

@@ -0,0 +1,143 @@
+"""choice-aware packaged OpenAI SSE orchestration."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import AsyncIterator
+from contextlib import suppress
+from typing import Any
+
+from flash.serve.runtime import (
+    StreamChoiceFinished,
+    StreamDelta,
+    StreamFinished,
+    StreamReady,
+)
+
+from .bootstrap import PublishedAdapter
+from .openai import (
+    ReasoningDeltaSplitter,
+    sse_data,
+    stream_chunk,
+    usage_stream_chunk,
+)
+
+
+async def stream_chat_body(
+    event_stream: AsyncIterator[Any],
+    ready: StreamReady,
+    resolved: PublishedAdapter,
+    provenance: dict[str, Any],
+    *,
+    choice_count: int,
+    include_usage: bool,
+) -> AsyncIterator[bytes]:
+    """render indexed choice events with one aggregate usage and done terminal."""
+
+    splitters = {
+        index: ReasoningDeltaSplitter(thinking=bool(ready.thinking))
+        for index in range(choice_count)
+    }
+    terminals: dict[int, StreamChoiceFinished] = {}
+    finished: StreamFinished | None = None
+    succeeded = False
+    try:
+        for index in range(choice_count):
+            yield sse_data(
+                stream_chunk(
+                    request_id=ready.request_id,
+                    model=resolved.requested_model,
+                    index=index,
+                    delta={"role": "assistant", "content": ""},
+                    provenance=provenance,
+                )
+            )
+        async for event in event_stream:
+            if type(event) is StreamDelta and finished is None:
+                if event.index in terminals or event.index not in splitters:
+                    raise RuntimeError("invalid stream choice delta")
+                rendered = splitters[event.index].feed(event.text)
+                if event.logprobs is not None and len(rendered) > 1:
+                    raise RuntimeError("a logprob delta crossed the reasoning boundary")
+                if not rendered and event.logprobs is not None:
+                    rendered = [("content", "")]
+                for position, (key, value) in enumerate(rendered):
+                    yield sse_data(
+                        stream_chunk(
+                            request_id=ready.request_id,
+                            model=resolved.requested_model,
+                            index=event.index,
+                            delta={key: value},
+                            logprobs=event.logprobs if position == 0 else None,
+                        )
+                    )
+                continue
+            if type(event) is StreamChoiceFinished and finished is None:
+                if event.index in terminals or event.index not in splitters:
+                    raise RuntimeError("invalid stream choice terminal")
+                terminals[event.index] = event
+                continue
+            if type(event) is StreamFinished and finished is None:
+                finished = event
+                continue
+            raise RuntimeError("invalid stream event order")
+        if finished is None or set(terminals) != set(range(choice_count)):
+            raise RuntimeError("stream ended without every choice terminal")
+        if (
+            finished.request_id != ready.request_id
+            or finished.runtime_id != ready.runtime_id
+            or finished.adapter_id != resolved.adapter.adapter_revision
+            or finished.incarnation != resolved.adapter.aggregate_sha256
+            or finished.thinking != ready.thinking
+        ):
+            raise RuntimeError("stream terminal identity mismatch")
+        for index in range(choice_count):
+            for key, value in splitters[index].finish():
+                yield sse_data(
+                    stream_chunk(
+                        request_id=ready.request_id,
+                        model=resolved.requested_model,
+                        index=index,
+                        delta={key: value},
+                    )
+                )
+            yield sse_data(
+                stream_chunk(
+                    request_id=ready.request_id,
+                    model=resolved.requested_model,
+                    index=index,
+                    delta={},
+                    finish_reason=terminals[index].finish_reason,
+                    provenance=provenance,
+                )
+            )
+        if include_usage:
+            yield sse_data(usage_stream_chunk(finished, resolved.requested_model, provenance))
+        succeeded = True
+    except asyncio.CancelledError:
+        raise
+    except Exception:
+        yield sse_data(
+            {
+                "error": {
+                    "message": "generation stream terminated",
+                    "type": "server_error",
+                    "code": "stream_terminated",
+                }
+            }
+        )
+    finally:
+        await close_iterator(event_stream)
+    if succeeded:
+        yield sse_data("[DONE]")
+
+
+async def close_iterator(iterator: AsyncIterator[Any]) -> None:
+    close = getattr(iterator, "aclose", None)
+    if close is None:
+        return
+    with suppress(Exception):
+        result = close()
+        if inspect.isawaitable(result):
+            await result

{freesolo_flash_dev-1.2.117 → freesolo_flash_dev-1.2.118}/flash/serve/app/http.py

@@ -5,9 +5,8 @@ from __future__ import annotations
 import asyncio
 import hashlib
 import hmac
-import inspect
 import json
-from collections.abc import AsyncIterator, Awaitable
+from collections.abc import Awaitable
 from contextlib import suppress
 from typing import Any
 
@@ -22,26 +21,43 @@ from flash.serve.runtime import (
     RuntimeConfigurationError,
     RuntimeNotReadyError,
     ServingRuntimeError,
-    StreamDelta,
-    StreamFinished,
     StreamReady,
 )
 
-from .bootstrap import PublishedAdapter, ServingBootstrap
+from .bootstrap import ServingBootstrap
+from .chat_stream import close_iterator, stream_chat_body
 from .openai import (
     OpenAIRequestError,
-    ReasoningDeltaSplitter,
     nonstream_response,
     parse_chat_request,
     provenance_payload,
-    sse_data,
-    stream_chunk,
-    usage_stream_chunk,
 )
 
 _REJECTED_AUTH_DIGEST = hashlib.sha256(b"flash-rejected-authorization").digest()
 
 
+async def _stream_body(
+    event_stream: Any,
+    ready: StreamReady,
+    resolved: Any,
+    provenance: dict[str, Any],
+    *,
+    include_usage: bool,
+    choice_count: int = 1,
+):
+    """preserve the historical packaged stream helper seam."""
+
+    async for chunk in stream_chat_body(
+        event_stream,
+        ready,
+        resolved,
+        provenance,
+        choice_count=choice_count,
+        include_usage=include_usage,
+    ):
+        yield chunk
+
+
 class _RequestBodyTooLarge(ValueError):
     """the observed request body exceeded the application byte ceiling."""
 
@@ -158,16 +174,16 @@ def create_app(
         try:
             first = await _await_until_disconnect(request, anext(event_stream))
         except asyncio.CancelledError:
-            await _close_iterator(event_stream)
+            await close_iterator(event_stream)
             raise
         except AdapterNotFoundError:
-            await _close_iterator(event_stream)
+            await close_iterator(event_stream)
             return _error(404, "model_not_found", "requested model is not deployed")
         except (PromptError, RuntimeConfigurationError):
-            await _close_iterator(event_stream)
+            await close_iterator(event_stream)
             return _error(400, "invalid_request", "generation request was rejected")
         except (EngineDeadError, RuntimeNotReadyError, ServingRuntimeError, Exception):
-            await _close_iterator(event_stream)
+            await close_iterator(event_stream)
             return _error(503, "service_unavailable", "generation service is unavailable")
         if (
             type(first) is not StreamReady
@@ -175,13 +191,14 @@ def create_app(
             or first.incarnation != resolved.adapter.aggregate_sha256
             or first.thinking != resolved.adapter.thinking_default
         ):
-            await _close_iterator(event_stream)
+            await close_iterator(event_stream)
             return _error(503, "service_unavailable", "generation stream did not become ready")
         body = _stream_body(
             event_stream,
             first,
             resolved,
             provenance,
+            choice_count=parsed.generation.n,
             include_usage=parsed.include_usage,
         )
         return StreamingResponse(body, media_type="text/event-stream", headers=headers)
@@ -312,96 +329,3 @@ def _error(
         status_code=status,
         headers=headers,
     )
-
-
-async def _stream_body(
-    event_stream: AsyncIterator[Any],
-    ready: StreamReady,
-    resolved: PublishedAdapter,
-    provenance: dict[str, Any],
-    *,
-    include_usage: bool,
-) -> AsyncIterator[bytes]:
-    splitter = ReasoningDeltaSplitter(thinking=bool(ready.thinking))
-    finished: StreamFinished | None = None
-    succeeded = False
-    try:
-        yield sse_data(
-            stream_chunk(
-                request_id=ready.request_id,
-                model=resolved.requested_model,
-                delta={"role": "assistant", "content": ""},
-                provenance=provenance,
-            )
-        )
-        async for event in event_stream:
-            if type(event) is StreamDelta and finished is None:
-                for key, value in splitter.feed(event.text):
-                    yield sse_data(
-                        stream_chunk(
-                            request_id=ready.request_id,
-                            model=resolved.requested_model,
-                            delta={key: value},
-                        )
-                    )
-                continue
-            if type(event) is StreamFinished and finished is None:
-                finished = event
-                continue
-            raise RuntimeError("invalid stream event order")
-        if finished is None or finished.finish_reason is None:
-            raise RuntimeError("stream ended without a real terminal event")
-        if (
-            finished.request_id != ready.request_id
-            or finished.runtime_id != ready.runtime_id
-            or finished.adapter_id != resolved.adapter.adapter_revision
-            or finished.incarnation != resolved.adapter.aggregate_sha256
-            or finished.thinking != ready.thinking
-        ):
-            raise RuntimeError("stream terminal identity mismatch")
-        for key, value in splitter.finish():
-            yield sse_data(
-                stream_chunk(
-                    request_id=ready.request_id,
-                    model=resolved.requested_model,
-                    delta={key: value},
-                )
-            )
-        yield sse_data(
-            stream_chunk(
-                request_id=ready.request_id,
-                model=resolved.requested_model,
-                delta={},
-                finish_reason=finished.finish_reason,
-                provenance=provenance,
-            )
-        )
-        if include_usage:
-            yield sse_data(usage_stream_chunk(finished, resolved.requested_model, provenance))
-        succeeded = True
-    except asyncio.CancelledError:
-        raise
-    except Exception:
-        yield sse_data(
-            {
-                "error": {
-                    "message": "generation stream terminated",
-                    "type": "server_error",
-                    "code": "stream_terminated",
-                }
-            }
-        )
-    finally:
-        await _close_iterator(event_stream)
-    if succeeded:
-        yield sse_data("[DONE]")
-
-
-async def _close_iterator(iterator: AsyncIterator[Any]) -> None:
-    close = getattr(iterator, "aclose", None)
-    if close is None:
-        return
-    with suppress(Exception):
-        result = close()
-        if inspect.isawaitable(result):
-            await result