freesolo-flash-dev 1.2.64__tar.gz → 1.2.65__tar.gz

{freesolo_flash_dev-1.2.64 → freesolo_flash_dev-1.2.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.64
+Version: 1.2.65
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.64 → freesolo_flash_dev-1.2.65}/flash/engine/worker/__init__.py

@@ -18,7 +18,7 @@ import traceback
 
 from flash._internal.diagnostics import sanitize_diagnostic
 from flash.adapters.fused_experts import lora_target_parameters
-from flash.core.spec import FIXED_SEED, load_job_spec_from_env
+from flash.core.spec import FIXED_SEED, gpu_count_of, load_job_spec_from_env
 from flash.engine.result.accounting import RunMetrics
 from flash.engine.worker.backend_common import collect_ray_failure_logs
 from flash.engine.worker.entry.opd import run_opd
@@ -80,7 +80,7 @@ from flash.engine.worker.perf import (
     grad_checkpointing_on,
     grpo_use_reentrant,
     is_cuda_oom,
-    preflight_free_vram,
+    preflight_gpu_occupancy,
     wait_for_gpu,
 )
 from flash.engine.worker.runtime.kernel_warmup import _current_cuda_sm, load_mega_cache
@@ -221,22 +221,16 @@ def _worker_failure_flags(exc: BaseException) -> dict[str, bool]:
     return {"retriable": retriable, "oom": (not retriable and is_cuda_oom(exc))}
 
 
-def _preflight_free_vram_for_spec() -> None:
-    """Reject a card that arrived already occupied by another tenant.
+def _preflight_gpu_occupancy_for_spec() -> None:
+    """Screen every GPU in the allocator-resolved worker allocation for substantial occupancy.
 
-    Nothing about the spec is read. An earlier draft re-derived the run's requirement here and
-    compared free bytes against it, which is a second sizing model competing with the allocator's
-    and wrong in both directions -- see ``preflight_free_vram``. Occupancy needs no requirement, so
-    it runs on any shape without reimplementing ``combined_vram_gb``'s non-linear multi-card fit.
+    The allocator stamps the chosen count onto ``JOB_SPEC`` before provider launch, so this passes
+    only the cards rented for this worker. It does not re-derive required VRAM or claim exact fit.
 
-    It samples device 0 only. On a multi-card node that leaves a tenant sitting on device 3 alone
-    undetected, which is a gap and not a false alarm -- device 0 occupancy is still evidence of a
-    shared host.
-
-    Must stay the first CUDA-adjacent call in boot: the reading is only trustworthy while this
+    This must stay the first CUDA-adjacent call in boot. The reading is trustworthy only while this
     process has no context of its own, and ``_force_fla_triton_gdn_on_sm100`` below creates one.
     """
-    preflight_free_vram()
+    preflight_gpu_occupancy(gpu_count_of(JOB_SPEC))
 
 
 THINKING = JOB_SPEC.thinking if JOB_SPEC else False
@@ -347,14 +341,14 @@ def _run_worker_mode() -> None:
     # FIRST, and specifically before `_force_fla_triton_gdn_on_sm100`: that reads
     # `get_device_capability`, which initializes CUDA in this process, and from that moment the
     # driver's "used" includes our own context with no sound way to subtract it -- the occupancy
-    # check declines to run rather than guess (see `preflight_free_vram`), so calling it later means
-    # never calling it at all. This is also before `_ensure_fla_fastpath_on_hopper`, whose repair
+    # check declines to run rather than guess (see `preflight_gpu_occupancy`), so calling it later
+    # means never calling it at all. this is also before `_ensure_fla_fastpath_on_hopper`, whose repair
     # path runs pip installs with 600s timeouts: a card handed over with a co-tenant's ~18GB still
     # resident fails the run either way, and the only question is whether that happens now or after
     # dependency repair, the model download and FSDP init have spent paid GPU on the same
     # conclusion. NVML answers without a context, so nothing here dirties the reading for the code
     # below.
-    _preflight_free_vram_for_spec()
+    _preflight_gpu_occupancy_for_spec()
     # run before model imports: sm100 tilelang GDN computes wrong gradients, so use Triton.
     _force_fla_triton_gdn_on_sm100()
     _ensure_fla_fastpath_on_hopper()

{freesolo_flash_dev-1.2.64 → freesolo_flash_dev-1.2.65}/flash/engine/worker/perf/__init__.py

@@ -28,7 +28,7 @@ from flash.engine.worker.perf.lifecycle import (
     RetriableInfraError,
     free_vram_gb,
     is_cuda_oom,
-    preflight_free_vram,
+    preflight_gpu_occupancy,
     total_vram_gb,
     wait_for_gpu,
 )
@@ -403,7 +403,7 @@ __all__ = [
     "grad_checkpointing_on",
     "grpo_use_reentrant",
     "is_cuda_oom",
-    "preflight_free_vram",
+    "preflight_gpu_occupancy",
     "total_vram_gb",
     "wait_for_gpu",
 ]

{freesolo_flash_dev-1.2.64 → freesolo_flash_dev-1.2.65}/flash/engine/worker/perf/lifecycle.py

@@ -34,36 +34,31 @@ class RetriableInfraError(RuntimeError):
 
 
 class DirtyGpuError(RetriableInfraError):
-    """The allocated card arrived with too little free VRAM to run on.
-
-    A RetriableInfraError, deliberately not an OOM. The two recover differently and only one of them
-    is right here: an OOM means the run does not fit the card and retries on a BIGGER one, spending
-    from a small dedicated OOM budget. A dirty card means the run fits fine and the card was already
-    occupied -- a co-tenant in another container, or a previous tenant's leak. Escalating size for
-    that pays more for a card that can be just as dirty, and burns an OOM retry proving it.
-
-    Infra retry reallocates instead: ``_handle_failure`` records the provider in ``failed_providers``
-    and the shape in ``tried_classes``, so the next attempt prefers a DIFFERENT provider, then an
-    untried shape, before clamping back. That is the right preference order for this failure --
-    co-tenancy is a property of the host pool that handed the instance out, not of the run -- but it
-    is a preference, not a guarantee, so nothing here promises which card comes back.
+    """An allocated GPU arrived with substantial pre-existing occupancy.
+
+    This is retriable infrastructure, deliberately not an OOM. An OOM means the run does not fit
+    and retries on a larger card. Substantial boot-time occupancy instead calls for a fresh
+    allocation, because a larger card can arrive just as occupied. This error does not claim that an
+    accepted card has enough free VRAM for the run: occupancy at or below 5%, unreadable NVML, and
+    invalid readings remain fail-open limits of the best-effort screen.
     """
 
 
-def _nvml_memory_gb() -> tuple[float, float] | None:
-    """``(free, total)`` on device 0 in GB straight from NVML, or None if NVML will not answer.
+def _nvml_memory_gb(device_index: int = 0) -> tuple[float, float] | None:
+    """``(free, total)`` for one device in GB from NVML, or None if NVML will not answer.
 
-    NVML, not ``torch.cuda.mem_get_info``: the torch call needs a CUDA context and CREATES one if the
-    process has none, so the act of measuring adds our own few hundred MB to the number being
-    measured. NVML queries the driver without initializing CUDA in this process, which is what lets
-    the boot reading be taken while the card is still provably untouched by us.
+    NVML, not ``torch.cuda.mem_get_info``: the torch call needs a CUDA context and creates one if the
+    process has none, so the act of measuring adds our own memory to the number being measured. NVML
+    queries the driver without initializing CUDA in this process. Device 0 remains the default for
+    the diagnostic helpers that expose its free and total memory.
     """
     try:
         import pynvml
 
         pynvml.nvmlInit()
         try:
-            info = pynvml.nvmlDeviceGetMemoryInfo(pynvml.nvmlDeviceGetHandleByIndex(0))
+            handle = pynvml.nvmlDeviceGetHandleByIndex(device_index)
+            info = pynvml.nvmlDeviceGetMemoryInfo(handle)
             return float(info.free) / (1024**3), float(info.total) / (1024**3)
         finally:
             with contextlib.suppress(Exception):
@@ -110,76 +105,44 @@ def total_vram_gb() -> float | None:
     return None if reading is None else reading[1]
 
 
-def preflight_free_vram(*, max_occupied_fraction: float = 0.05) -> None:
-    """Fail fast when the allocated card arrives already occupied by somebody else.
-
-    Flash sizes the run, asks the provider for a card that fits, and then trains on whatever comes
-    back without ever checking that the memory is FREE. A real RTX 4090 arrived with
-    ``total=22.5 used=18.6 free=3.4``, only 0.486 GB of it owned by any process in this container:
-    ~18 GB was a co-tenant in another container. The number was already in the first heartbeat,
-    before any work -- recorded and not acted on -- so the run downloaded the model and built FSDP
-    for ~80s of paid GPU before dying on an OOM that was knowable at boot.
-
-    Measured as OCCUPANCY (``used/total`` as the driver reports it), not against a requirement.
-    Re-deriving "what this run needs" here would be a second sizing model competing with the
-    allocator's, and it loses both ways: the allocator sizes SFT from profile-measured knobs
-    (``_overridden_train``) that reduce an authored batch 8 to the executed batch 1, so recomputing
-    from the authored spec demands more than the card was rented for; and a run sized exactly at a
-    catalog tier (24 GB) can exceed a real card's usable total (22.5 GB on a 4090), which no amount
-    of free memory can ever satisfy. Both reject a perfectly clean card and retry until the infra
-    budget is gone -- strictly worse than the OOM this exists to prevent.
-
-    Occupancy has neither failure mode: it needs no requirement, no profile, and no catalog number.
-    An empty card reads ~0 whatever is scheduled on it, so the threshold can sit near zero rather
-    than being padded to cover a requirement it never consults. 5% is the driver's own reserve plus
-    slack; the observed dirty card was at 83% with 0.486 GB attributable to us.
-
-    The floor is low BECAUSE the check is occupancy: a loose threshold has to be justified against
-    the biggest run that could be scheduled, and a 5 GB co-tenant that leaves a 20 GB run 17.5 GB on
-    a 22.5 GB card is 22% occupied and fatal. Sizing the threshold to survive that means re-deriving
-    the requirement, which is the failure above. Refusing every card with a stranger on it needs no
-    requirement and covers the close-fitting case too.
-
-    A threshold that low only works if none of the used memory is OURS, and that is established by
-    WHEN the reading is taken, not by trying to attribute it afterwards. Attribution was tried and
-    cannot work here: ``nvidia-smi --query-compute-apps`` reports HOST pids, while the worker
-    container runs with a private pid namespace (``docker run`` in ``providers/_lifecycle/instance``
-    passes no ``--pid=host``), so testing ``/proc/<pid>`` inside the container is not an ownership
-    test in either direction -- our own rows fail it and get counted as a stranger's, and under
-    ``--pid=host`` a real co-tenant passes it and gets credited to us, silently waving through the
-    exact card this exists to refuse.
-
-    So the check simply declines to run once this process has a CUDA context, which makes WHEN it
-    runs the whole design. There is exactly one call site, ``_preflight_free_vram_for_spec`` at the
-    top of ``_run_worker_mode``, and it is placed before ``_force_fla_triton_gdn_on_sm100`` because
-    that reads ``get_device_capability`` and creates the process's first context. Calling from
-    anywhere below that point is not a weaker check, it is no check at all: the guard above returns
-    immediately and the card is never read. A single early call site is therefore the invariant, not
-    an omission -- a later retry could only run after CUDA came up, when the reading is already
-    unusable. That also means one NVML failure at boot is one lost reading, not a lost run: the
-    guard below returns and training proceeds exactly as it did before this check existed.
+def preflight_gpu_occupancy(gpu_count: int, *, max_occupied_fraction: float = 0.05) -> None:
+    """Screen each allocated GPU for substantial occupancy before CUDA initialization.
+
+    The worker reads exactly the allocator-resolved device indices ``range(gpu_count)`` through NVML.
+    An unreadable device or a non-positive total is skipped without hiding later allocated devices.
+    If any valid reading exceeds 5% occupancy, the allocation is rejected as retriable
+    infrastructure after every allocated device has been inspected.
+
+    This is a best-effort occupancy boundary, not an exact-fit guarantee. It deliberately does not
+    reconstruct the allocator's run requirement. Occupancy at or below 5%, unreadable NVML, and
+    invalid readings remain accepted. Once this process has initialized CUDA, the whole screen is
+    fail-open because the driver reading includes memory owned by this process with no sound way to
+    subtract it.
     """
     if cuda_is_initialized():
-        # some of `used` would be ours and there is no sound way to tell how much. a check that
-        # cannot distinguish our context from a co-tenant would either false-reject clean cards or
-        # need a threshold too loose to catch the tenant it exists for. staying silent is correct:
-        # the boot call already read this card while it was provably untouched.
+        # some of `used` would be ours and there is no sound way to tell how much
         return
-    reading = _nvml_memory_gb()
-    if reading is None:
-        # NVML would not answer. somebody else's failure to report, not evidence of a dirty card --
-        # training fails soon enough with a better message.
-        return
-    free, total = reading
-    if total <= 0:
-        return
-    used = max(0.0, total - free)
-    if used <= total * max_occupied_fraction:
+
+    first_dirty: tuple[int, float, float] | None = None
+    for device_index in range(gpu_count):
+        reading = _nvml_memory_gb(device_index)
+        if reading is None:
+            # one unreadable device is not evidence about any later allocated device
+            continue
+        free, total = reading
+        if total <= 0:
+            continue
+        used = max(0.0, total - free)
+        if used > total * max_occupied_fraction and first_dirty is None:
+            first_dirty = (device_index, used, total)
+
+    if first_dirty is None:
         return
+    device_index, used, total = first_dirty
     raise DirtyGpuError(
-        f"allocated GPU has {used:.1f} GB of {total:.1f} GB ({used / total:.0%}) already in use "
-        "before this run has touched it; the card is occupied by another tenant or a previous "
-        "tenant's leak, so retrying on a freshly allocated instance"
+        f"allocated GPU device {device_index} has {used:.1f} GB of {total:.1f} GB "
+        f"({used / total:.0%}) already in use before this run has touched it; the card is occupied "
+        "by another tenant or a previous tenant's leak, so retrying on a freshly allocated instance"
     )
 
 

{freesolo_flash_dev-1.2.64 → freesolo_flash_dev-1.2.65}/flash/server/domain/projects.py

@@ -58,17 +58,11 @@ def _project_payload_id(payload: Any) -> str:
 
 
 def _payload_project_slug(payload: Any) -> str:
-    """The project slug carried by a validation response, or ``""``.
+    """Return the authoritative project slug carried by a validation response, or ``""``.
 
-    Accepted at the top level or nested under ``project``, and under either spelling, because the
-    internal and user-key validation endpoints return differently shaped bodies.
-
-    The project directory stores no slug column, so no deployment actually sends one of those
-    fields today: the slug is derived from the project's name instead, which the directory does
-    return and holds ``UNIQUE (org_id, name)``. That uniqueness is what makes the derivation a
-    stable identity rather than a guess -- two projects in one org cannot normalize to the same
-    directory. An explicit field still wins when a backend grows one, so this needs no second
-    change to hand authority back.
+    The internal and user-key validation endpoints return differently shaped bodies, so accept the
+    canonical field at the top level or nested under ``project`` and under either current spelling.
+    Project display names are not identity and are never used as a fallback.
     """
     if not isinstance(payload, dict):
         return ""
@@ -81,28 +75,9 @@ def _payload_project_slug(payload: Any) -> str:
             value = source.get(field)
             if isinstance(value, str) and value.strip():
                 return value.strip()
-    for source in candidates:
-        for field in ("projectName", "name"):
-            derived = _slug_from_project_name(source.get(field))
-            if derived:
-                return derived
     return ""
 
 
-def _slug_from_project_name(value: Any) -> str:
-    """The hub directory segment a project ``name`` normalizes to, or ``""``.
-
-    Shares :func:`normalize_env_name_segment` with the name segment so both halves of a published
-    ``<org>/<project>/<name>`` id obey one grammar, and so a name that survives normalization here
-    is one ``publish_slug_for_name`` will accept rather than reject a step later.
-    """
-    from flash.schema import normalize_env_name_segment
-
-    if not isinstance(value, str) or not value.strip():
-        return ""
-    return normalize_env_name_segment(value) or ""
-
-
 def _internal_http_error(*, status: int, raw: bytes, project_id: str) -> HTTPException:
     detail = _raw_error_detail(raw)
     if status == 404:
@@ -191,20 +166,11 @@ def require_project_access_slug(
     authorization: str | None,
     org_id: str | None = None,
 ) -> tuple[str, str]:
-    """As :func:`require_project_access`, but also returns the project's non-empty slug.
-
-    The slug is the second segment of every environment slug this project publishes, so a caller
-    that asked for one cannot proceed without it. Validation therefore ends here rather than at
-    the publish itself: the destination is unknowable, and the failure has to name that.
-
-    Two ways a slug can come back empty, both of which used to reach ``publish_slug_for_name``
-    and surface as "re-run `flash login` to refresh the key" -- advice that cannot help, because
-    neither cause is the caller's key. A standalone plane has no Freesolo project directory to
-    resolve a slug from and cannot publish to the managed hub at all; on a managed plane the slug
-    is derived from the project's name, so the only remaining way to reach here is a name with no
-    character that survives normalization (``"???"``). Reporting them apart is the point: one is a
-    permanent property of the deployment, the other is fixed by renaming the project -- and the
-    second is actionable by the caller, so it must not be reported as an opaque upstream fault.
+    """Validate project access and return its authoritative non-empty slug.
+
+    User keys always pass through the public bearer-authenticated project endpoint first so project
+    scoping is enforced. If that response omits the canonical slug, the existing internal endpoint
+    resolves it for the already-approved project and the verified organization.
     """
     project_id, slug = _project_access(
         project_id=project_id,
@@ -212,23 +178,36 @@ def require_project_access_slug(
         authorization=authorization,
         org_id=org_id,
     )
-    if not slug:
-        if standalone():
+    if slug:
+        return project_id, slug
+    if standalone():
+        raise HTTPException(
+            status_code=501,
+            detail=(
+                "publishing environments requires a Freesolo project directory to resolve "
+                "the project slug, which a standalone plane does not have; reference the "
+                "environment from git instead (see SELF_HOSTING.md)"
+            ),
+        )
+
+    if key.get("auth_kind") != "internal":
+        expected_org = str(key.get("org_id") or "").strip()
+        if not expected_org:
             raise HTTPException(
-                status_code=501,
+                status_code=502,
                 detail=(
-                    "publishing environments requires a Freesolo project directory to resolve "
-                    "the project slug, which a standalone plane does not have; reference the "
-                    "environment from git instead (see SELF_HOSTING.md)"
+                    "authenticated Freesolo key is missing the organization id required to "
+                    "resolve the canonical project slug"
                 ),
             )
+        _, slug = _require_internal_project_access(
+            project_id=project_id,
+            org_id=expected_org,
+        )
+    if not slug:
         raise HTTPException(
-            status_code=400,
-            detail=(
-                f"project {project_id!r} has no name that can form a publishable slug, so the "
-                "environment's destination cannot be resolved; rename the project to include "
-                "a letter or digit"
-            ),
+            status_code=502,
+            detail="Freesolo project validation did not return a canonical project slug",
         )
     return project_id, slug
 

{freesolo_flash_dev-1.2.64 → freesolo_flash_dev-1.2.65}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.64"
+version = "1.2.65"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -137,7 +137,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.64"
+version = "1.2.65"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]