freesolo-flash-dev 1.4.13__tar.gz → 1.4.14__tar.gz

{freesolo_flash_dev-1.4.13 → freesolo_flash_dev-1.4.14}/.github/workflows/dev-kernel-cache.yml

@@ -2,6 +2,7 @@ name: dev kernel-cache candidate
 
 # dev pushes only run the metadata detector. the scheduled or manual run builds the newest
 # missing candidate, so several daytime changes coalesce into one off-peak gpu bake.
+# a dev -> main promotion builds missing candidates immediately and retries its readiness gate.
 
 on:
   push:
@@ -11,11 +12,12 @@ on:
       - .github/workflows/worker-image.yml
       - .github/workflows/bake-kernel-cache.yml
       - .github/workflows/dev-kernel-cache.yml
-      - docker/kernel_fingerprint.py
-      - docker/make_rp_handler.py
-      - docker/extract_cudart_fix.py
+      - docker/**
       - flash/**
       - pyproject.toml
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
   schedule:
     # 09:00 utc is 01:00 pst / 02:00 pdt.
     - cron: "0 9 * * *"
@@ -33,7 +35,15 @@ permissions:
   packages: write
 
 concurrency:
-  group: dev-kernel-cache
+  # metadata pushes and rejected PRs must not replace a queued bake in the pending slot.
+  group: >-
+    dev-kernel-cache-${{
+      (github.event_name == 'workflow_dispatch' ||
+       (github.event_name == 'pull_request' &&
+        github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event.pull_request.head.ref == 'dev' && !github.event.pull_request.draft))
+      && 'build' || 'detect'
+    }}
   cancel-in-progress: false
 
 jobs:
@@ -48,7 +58,11 @@ jobs:
         run: gh workflow run dev-kernel-cache.yml --repo "$GITHUB_REPOSITORY" --ref dev
 
   detect:
-    if: github.repository == 'freesolo-co/flash' && github.event_name != 'schedule'
+    if: >-
+      github.repository == 'freesolo-co/flash' && github.event_name != 'schedule' &&
+      (github.event_name != 'pull_request' ||
+       (github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event.pull_request.head.ref == 'dev' && !github.event.pull_request.draft))
     runs-on: ubuntu-latest
     timeout-minutes: 15
     outputs:
@@ -63,7 +77,7 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Resolve source metadata
         id: source
@@ -143,7 +157,7 @@ jobs:
 
   build-base:
     needs: detect
-    if: github.event_name == 'workflow_dispatch' && needs.detect.outputs.needs_bake == 'true'
+    if: (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && needs.detect.outputs.needs_bake == 'true'
     uses: ./.github/workflows/worker-image.yml
     with:
       tag: cu128-dev-${{ needs.detect.outputs.revision }}
@@ -152,11 +166,11 @@ jobs:
 
   bake:
     needs: [detect, build-base]
-    if: github.event_name == 'workflow_dispatch' && needs.detect.outputs.needs_bake == 'true'
+    if: (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && needs.detect.outputs.needs_bake == 'true'
     uses: ./.github/workflows/bake-kernel-cache.yml
     with:
       sms: ${{ needs.detect.outputs.sms }}
-      deadline_minutes: ${{ fromJSON(format('{0}', inputs.deadline_minutes)) }}
+      deadline_minutes: ${{ fromJSON(github.event_name == 'pull_request' && '90' || format('{0}', inputs.deadline_minutes)) }}
       fp_cache: ${{ needs.build-base.outputs.fp_cache }}
       fp_base: ${{ needs.build-base.outputs.fp_base }}
       base_ref: ${{ needs.build-base.outputs.base_ref }}
@@ -166,7 +180,7 @@ jobs:
 
   publish-ready:
     needs: [detect, bake]
-    if: github.event_name == 'workflow_dispatch' && needs.detect.outputs.needs_bake == 'true'
+    if: (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && needs.detect.outputs.needs_bake == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -213,7 +227,26 @@ jobs:
               "$repo:cu128-dev-ready-$sm"
           done
 
-      - name: Re-run production readiness for dev
+  refresh-readiness:
+    needs: [detect, publish-ready]
+    # a queued revision may find compatible caches published by the preceding bake.
+    # always() permits the skipped publication dependency; failed/cancelled work stays red.
+    if: >-
+      always() && !cancelled() && needs.detect.result == 'success' &&
+      (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') &&
+      (needs.detect.outputs.needs_bake == 'false' || needs['publish-ready'].result == 'success')
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Re-run failed production readiness for this revision
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: gh workflow run production-kernel-cache-ready.yml --repo "$GITHUB_REPOSITORY" --ref dev
+          REVISION: ${{ needs.detect.outputs.revision }}
+        run: |
+          set -euo pipefail
+          runs=$(gh run list --repo "$GITHUB_REPOSITORY" \
+            --workflow production-kernel-cache-ready.yml --commit "$REVISION" \
+            --event pull_request --status failure --json databaseId --jq '.[].databaseId')
+          for run_id in $runs; do
+            gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY" --failed
+          done

{freesolo_flash_dev-1.4.13 → freesolo_flash_dev-1.4.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.4.13
+Version: 1.4.14
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.4.13 → freesolo_flash_dev-1.4.14}/flash/envs/loading/staged.py

@@ -75,6 +75,7 @@ def _reference_for_environment_id(env_id: str) -> tuple[str, Any]:
 
 def is_staged_environment_transient_error(exc: BaseException) -> bool:
     """Classify timeout, connection, rate-limit, and server failures across clients."""
+    from flash.artifacts.errors import artifact_error_transience
     from flash.envs.meta.identity import GitHubTransientError
     from flash.envs.registry.azure import RegistryError
     from flash.providers.artifacts.channel import artifact_status_code
@@ -87,6 +88,9 @@ def is_staged_environment_transient_error(exc: BaseException) -> bool:
             return True
         if isinstance(current, (StagedEnvironmentTransientError, GitHubTransientError)):
             return True
+        transience = artifact_error_transience(current)
+        if transience is not None:
+            return transience
         status = artifact_status_code(current)
         if status == 429 or (status is not None and status >= 500):
             return True

{freesolo_flash_dev-1.4.13 → freesolo_flash_dev-1.4.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.4.13"
+version = "1.4.14"
 description ="Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # The training and serving stack is validated on Python 3.11 and 3.12.
@@ -211,7 +211,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.4.13"
+version = "1.4.14"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

freesolo_flash_dev-1.4.14/tests/test_kernel_cache_promotion.py

@@ -0,0 +1,202 @@
+"""Promotion-triggered bakes must be trusted, bounded, and refresh the original PR check."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOW = yaml.safe_load((ROOT / ".github/workflows/dev-kernel-cache.yml").read_text())
+JOBS = WORKFLOW["jobs"]
+
+
+def _evaluate(expression, **context):
+    # these workflow conditions use only boolean operators, comparisons, and field access.
+    expression = expression.removeprefix("${{").removesuffix("}}").strip()
+    expression = expression.replace("&&", " and ").replace("||", " or ")
+    expression = re.sub(r"!(?!=)", " not ", expression)
+    return eval(
+        f"({expression})",
+        {"__builtins__": {}, "fromJSON": json.loads, "format": str.format},
+        context,
+    )
+
+
+def _github(event, *, repo="freesolo-co/flash", head="dev", draft=False):
+    return SimpleNamespace(
+        repository="freesolo-co/flash",
+        event_name=event,
+        event=SimpleNamespace(
+            pull_request=SimpleNamespace(
+                head=SimpleNamespace(repo=SimpleNamespace(full_name=repo), ref=head), draft=draft
+            )
+        ),
+    )
+
+
+@pytest.mark.parametrize(
+    ("event", "head_repo", "head", "draft", "allowed"),
+    [
+        ("push", "freesolo-co/flash", "dev", False, True),
+        ("workflow_dispatch", "freesolo-co/flash", "dev", False, True),
+        ("schedule", "freesolo-co/flash", "dev", False, False),
+        ("pull_request", "freesolo-co/flash", "dev", False, True),
+        ("pull_request", "someone/flash", "dev", False, False),
+        ("pull_request", "freesolo-co/flash", "feature", False, False),
+        ("pull_request", "freesolo-co/flash", "dev", True, False),
+    ],
+)
+def test_only_trusted_nondraft_promotions_enter_candidate_workflow(
+    event, head_repo, head, draft, allowed
+):
+    triggers = WORKFLOW.get("on", WORKFLOW.get(True))
+    assert triggers["pull_request"]["branches"] == ["main"]
+    assert "ready_for_review" in triggers["pull_request"]["types"]
+    assert (
+        _evaluate(
+            JOBS["detect"]["if"], github=_github(event, repo=head_repo, head=head, draft=draft)
+        )
+        is allowed
+    )
+    group = WORKFLOW["concurrency"]["group"].removeprefix("dev-kernel-cache-")
+    assert _evaluate(group, github=_github(event, repo=head_repo, head=head, draft=draft)) == (
+        "build" if allowed and event in {"workflow_dispatch", "pull_request"} else "detect"
+    )
+
+
+@pytest.mark.parametrize("event", ["push", "workflow_dispatch", "pull_request"])
+@pytest.mark.parametrize("missing", [False, True])
+def test_gpu_jobs_run_only_for_missing_manual_or_promotion_caches(event, missing):
+    needs = SimpleNamespace(
+        detect=SimpleNamespace(outputs=SimpleNamespace(needs_bake=str(missing).lower()))
+    )
+    for name in ("build-base", "bake", "publish-ready"):
+        assert "detect" in JOBS[name]["needs"]
+        assert _evaluate(JOBS[name]["if"], github=_github(event), needs=needs) is (
+            missing and event != "push"
+        )
+
+
+def test_metadata_push_cannot_displace_a_queued_bake():
+    group = WORKFLOW["concurrency"]["group"].removeprefix("dev-kernel-cache-")
+    assert WORKFLOW["concurrency"]["cancel-in-progress"] is False
+    assert _evaluate(group, github=_github("push")) == "detect"
+    assert _evaluate(group, github=_github("schedule")) == "detect"
+    for event in ("workflow_dispatch", "pull_request"):
+        assert _evaluate(group, github=_github(event)) == "build"
+
+
+@pytest.mark.parametrize(
+    ("event", "deadline", "expected"),
+    [("pull_request", "", 90), ("workflow_dispatch", 30, 30), ("workflow_dispatch", 0, 0)],
+)
+def test_promotion_has_bounded_deadline_without_masking_invalid_manual_zero(
+    event, deadline, expected
+):
+    assert (
+        _evaluate(
+            JOBS["bake"]["with"]["deadline_minutes"],
+            github=_github(event),
+            inputs=SimpleNamespace(deadline_minutes=deadline),
+        )
+        == expected
+    )
+
+
+@pytest.mark.parametrize("run_ids", ["", "123\n456\n"])
+def test_ready_publication_retries_failed_pr_checks_for_exact_baked_revision(tmp_path, run_ids):
+    step = next(
+        step
+        for step in JOBS["refresh-readiness"]["steps"]
+        if step.get("name", "").startswith("Re-run")
+    )
+    assert step["env"]["REVISION"] == "${{ needs.detect.outputs.revision }}"
+    gh = tmp_path / "gh"
+    gh.write_text(
+        "#!/usr/bin/env python3\n"
+        "import json, os, sys\n"
+        "with open(os.environ['CALLS'], 'a') as f:\n"
+        "    f.write(json.dumps(sys.argv[1:]) + '\\n')\n"
+        "if sys.argv[1:3] == ['run', 'list']:\n"
+        "    print(os.environ['RUN_IDS'])\n"
+    )
+    gh.chmod(0o755)
+    log = tmp_path / "calls"
+    revision = "a" * 40
+    subprocess.run(
+        ["bash", "-eu", "-c", step["run"]],
+        check=True,
+        env={
+            **os.environ,
+            "PATH": f"{tmp_path}:{os.environ['PATH']}",
+            "CALLS": str(log),
+            "RUN_IDS": run_ids,
+            "REVISION": revision,
+            "GITHUB_REPOSITORY": "freesolo-co/flash",
+        },
+    )
+    calls = [json.loads(line) for line in log.read_text().splitlines()]
+    assert calls[0] == [
+        "run",
+        "list",
+        "--repo",
+        "freesolo-co/flash",
+        "--workflow",
+        "production-kernel-cache-ready.yml",
+        "--commit",
+        revision,
+        "--event",
+        "pull_request",
+        "--status",
+        "failure",
+        "--json",
+        "databaseId",
+        "--jq",
+        ".[].databaseId",
+    ]
+    assert calls[1:] == [
+        ["run", "rerun", run_id, "--repo", "freesolo-co/flash", "--failed"]
+        for run_id in run_ids.split()
+    ]
+
+
+@pytest.mark.parametrize("event", ["pull_request", "workflow_dispatch", "push"])
+@pytest.mark.parametrize(
+    ("detected", "missing", "published", "cancelled", "refresh"),
+    [
+        ("success", "true", "success", False, True),
+        # B was queued behind A; A has now published a compatible cache for B.
+        ("success", "false", "skipped", False, True),
+        ("success", "true", "failure", False, False),
+        ("success", "true", "skipped", False, False),
+        ("failure", "false", "skipped", False, False),
+        ("skipped", "false", "skipped", False, False),
+        ("success", "false", "skipped", True, False),
+    ],
+)
+def test_readiness_refresh_handles_queued_cache_hits_and_failed_dependencies(
+    event, detected, missing, published, cancelled, refresh
+):
+    class Needs(SimpleNamespace):
+        def __getitem__(self, key):
+            return getattr(self, key)
+
+    job = JOBS["refresh-readiness"]
+    assert job["needs"] == ["detect", "publish-ready"]
+    assert _evaluate(
+        job["if"],
+        github=_github(event),
+        needs=Needs(
+            detect=SimpleNamespace(result=detected, outputs=SimpleNamespace(needs_bake=missing)),
+            **{"publish-ready": SimpleNamespace(result=published)},
+        ),
+        always=lambda: True,
+        cancelled=lambda: cancelled,
+    ) is (refresh and event != "push")

{freesolo_flash_dev-1.4.13 → freesolo_flash_dev-1.4.14}/tests/test_staged_environment.py

@@ -377,6 +377,69 @@ def test_missing_immutable_revision_is_terminal(monkeypatch, tmp_path) -> None:
         )
 
 
+@pytest.mark.parametrize("stage", ["manifest", "archive"])
+@pytest.mark.parametrize("wrapper", [None, "__cause__", "__context__"])
+@pytest.mark.parametrize(
+    ("failure_kind", "retriable"),
+    [
+        ("request", True),
+        ("response", True),
+        (408, True),
+        (409, True),
+        (412, True),
+        (429, True),
+        (503, True),
+        (400, False),
+        (401, False),
+        (403, False),
+        (404, False),
+    ],
+)
+def test_azure_hydration_failure_preserves_worker_retry_budget(
+    monkeypatch, tmp_path, stage, wrapper, failure_kind, retriable
+) -> None:
+    from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError
+
+    from flash.envs.loading.staged import materialize_staged_environment
+
+    package, manifest, archive, manifest_path, archive_path = _staged_files(tmp_path)
+    if failure_kind == "request":
+        failure = ServiceRequestError("transport interrupted")
+    elif failure_kind == "response":
+        # no timeout/connection-named cause: the Azure type alone must be enough.
+        failure = ServiceResponseError("transport interrupted")
+    else:
+        failure = HttpResponseError("artifact request failed")
+        failure.status_code = failure_kind
+    if wrapper:
+        outer = RuntimeError("download failed")
+        setattr(outer, wrapper, failure)
+        failure = outer
+    calls = []
+    failed_path = manifest_path if stage == "manifest" else archive_path
+
+    def download(**kwargs):
+        filename = kwargs["filename"]
+        calls.append(filename)
+        if filename == failed_path:
+            raise failure
+        return {manifest_path: str(manifest), archive_path: str(archive)}[filename]
+
+    monkeypatch.setattr("flash.artifacts.api.download_file", download)
+    with pytest.raises(RuntimeError) as raised:
+        materialize_staged_environment(
+            _spec(package=package).environment,
+            hf_repo="owner/run-artifacts",
+        )
+    assert calls == ([manifest_path] if stage == "manifest" else [manifest_path, archive_path])
+    assert isinstance(raised.value, StagedEnvironmentTransientError) is retriable
+    assert raised.value.__cause__ is failure
+    assert worker_entry._worker_failure_flags(raised.value) == {
+        "retriable": retriable,
+        "oom": False,
+    }
+
+
 def test_worker_failure_boundary_marks_staged_transient_retriable() -> None:
     from flash.envs.meta.identity import GitHubUnavailableError
 

{freesolo_flash_dev-1.4.13 → freesolo_flash_dev-1.4.14}/tests/test_workflow_contracts.py

@@ -506,7 +506,9 @@ def test_dev_cache_pushes_only_detect_and_off_peak_runs_build():
         for step in _steps(jobs["detect"])
         if str(step.get("uses", "")).startswith("actions/checkout@")
     )
-    assert detect_checkout["with"]["ref"] == "${{ github.sha }}"
+    assert (
+        detect_checkout["with"]["ref"] == "${{ github.event.pull_request.head.sha || github.sha }}"
+    )
 
     build = jobs["build-base"]
     bake = jobs["bake"]
@@ -518,9 +520,9 @@ def test_dev_cache_pushes_only_detect_and_off_peak_runs_build():
     assert bake["uses"] == "./.github/workflows/bake-kernel-cache.yml"
     assert "cu128-dev-" in bake["with"]["target_tag_prefix"]
     assert triggers["workflow_dispatch"]["inputs"]["deadline_minutes"]["default"] == 90
-    assert "fromJSON(format(" in bake["with"]["deadline_minutes"]
+    assert "fromJSON(" in bake["with"]["deadline_minutes"]
     assert "inputs.deadline_minutes" in bake["with"]["deadline_minutes"]
-    assert "||" not in bake["with"]["deadline_minutes"]  # zero must fail validation, not become 90
+    assert "inputs.deadline_minutes ||" not in bake["with"]["deadline_minutes"]
     bake_workflow = _load(WORKFLOW_DIR / "bake-kernel-cache.yml")
     assert "deadline_minutes" in _triggers(bake_workflow)["workflow_call"]["inputs"]
     warm = next(
@@ -540,12 +542,14 @@ def test_dev_cache_pushes_only_detect_and_off_peak_runs_build():
     assert script.index("candidate validation failed") < script.index("crane copy")
     rerun = next(
         step
-        for step in _steps(publish)
-        if step.get("name") == "Re-run production readiness for dev"
+        for step in _steps(jobs["refresh-readiness"])
+        if step.get("name") == "Re-run failed production readiness for this revision"
     )["run"]
     assert "production-kernel-cache-ready.yml" in rerun
     assert '--repo "$GITHUB_REPOSITORY"' in rerun
-    assert "--ref dev" in rerun
+    assert '--commit "$REVISION"' in rerun
+    assert "--event pull_request --status failure" in rerun
+    assert "gh run rerun" in rerun
 
     classify = next(step for step in _steps(jobs["detect"]) if step.get("id") == "classify")["run"]
     assert '[ "$prod_pc" != "$FP_CACHE" ] && [ "$candidate_pc" != "$FP_CACHE" ]' in classify

{freesolo_flash_dev-1.4.13 → freesolo_flash_dev-1.4.14}/uv.lock

@@ -1297,7 +1297,7 @@ wheels = [
 
 [[package]]
 name = "freesolo-flash"
-version = "1.4.13"
+version = "1.4.14"
 source = { editable = "." }
 
 [package.optional-dependencies]