freesolo-flash-dev 1.2.67__tar.gz → 1.2.68__tar.gz

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.67
+Version: 1.2.68
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.68}/flash/cli/commands/env/push.py

@@ -716,7 +716,20 @@ def cmd_env_push(args) -> int:
         readme = pkg / "README.md"
         if not readme.exists():
             readme.write_text(f"# {env_name}\n\nFlash Freesolo environment.\n")
-        # One progress widget spans both phases the user otherwise waits through silently:
+        from flash.envs.package.direct_tokens import (
+            DirectTokenScanError,
+            package_contains_direct_token,
+        )
+
+        try:
+            contains_direct_token = package_contains_direct_token(pkg)
+        except DirectTokenScanError:
+            return _err("environment package could not be scanned safely")
+        if contains_direct_token:
+            return _err(
+                "environment package contains a direct access token; remove it before publishing"
+            )
+        # one progress widget spans both phases the user otherwise waits through silently:
         # packaging (walk + gzip, slow for large datasets) and the upload itself.
         bar = _UploadProgress(env_name)
         bar.status("packaging environment")

freesolo_flash_dev-1.2.68/flash/envs/package/direct_tokens.py

@@ -0,0 +1,136 @@
+"""Bounded raw-byte scanning for direct environment package tokens."""
+
+from __future__ import annotations
+
+import os
+import re
+import stat
+from collections.abc import Iterator
+from pathlib import Path
+
+_CHUNK_SIZE = 64 * 1024
+_URLSAFE_CLASS = rb"A-Za-z0-9_-"
+_ALPHANUMERIC_CLASS = rb"A-Za-z0-9"
+_TOKEN_RULES = (
+    (b"fslo_", _URLSAFE_CLASS, 45),
+    (b"hf_", _ALPHANUMERIC_CLASS, 34),
+    (b"pit_", _ALPHANUMERIC_CLASS, 64),
+)
+_TOKEN_PATTERNS = tuple(
+    re.compile(
+        rb"(?<!["
+        + _URLSAFE_CLASS
+        + rb"])(?:"
+        + re.escape(prefix)
+        + rb")(["
+        + body_class
+        + rb"]{"
+        + str(body_length).encode()
+        + rb"})(?!["
+        + _URLSAFE_CLASS
+        + rb"])"
+    )
+    for prefix, body_class, body_length in _TOKEN_RULES
+)
+_MAX_TOKEN_SIZE = max(len(prefix) + body_length for prefix, _, body_length in _TOKEN_RULES)
+_OVERLAP = _MAX_TOKEN_SIZE + 1
+_PLACEHOLDER_BODIES = frozenset(
+    {
+        b"exampletoken",
+        b"notarealtoken",
+        b"placeholdertoken",
+        b"replacemewithtoken",
+        b"testtoken",
+        b"yourapitokenhere",
+        b"yourtokenhere",
+    }
+)
+
+
+class DirectTokenScanError(Exception):
+    """A package could not be scanned without following or skipping a member."""
+
+    def __init__(self) -> None:
+        super().__init__("package scan failed")
+
+
+def _is_obvious_placeholder(body: bytes) -> bool:
+    normalized = body.lower().translate(None, b"_-")
+    if normalized in _PLACEHOLDER_BODIES:
+        return True
+    return bool(normalized) and normalized[:1] in {b"0", b"x"} and len(set(normalized)) == 1
+
+
+def _buffer_contains_direct_token(data: bytes, *, start_at: int, finalized_start_end: int) -> bool:
+    for pattern in _TOKEN_PATTERNS:
+        for match in pattern.finditer(data):
+            if match.start() < start_at:
+                continue
+            if match.start() >= finalized_start_end:
+                break
+            if not _is_obvious_placeholder(match.group(1)):
+                return True
+    return False
+
+
+def _file_contains_direct_token(path: Path) -> bool:
+    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
+    fd: int | None = os.open(path, flags)
+    try:
+        if not stat.S_ISREG(os.fstat(fd).st_mode):
+            raise DirectTokenScanError
+        with os.fdopen(fd, "rb") as stream:
+            fd = None
+            data = b""
+            buffer_start = 0
+            finalized_start = 0
+            while chunk := stream.read(_CHUNK_SIZE):
+                data += chunk
+                start_at = finalized_start - buffer_start
+                finalized_start_end = max(start_at, len(data) - _MAX_TOKEN_SIZE)
+                if _buffer_contains_direct_token(
+                    data,
+                    start_at=start_at,
+                    finalized_start_end=finalized_start_end,
+                ):
+                    return True
+                finalized_start = buffer_start + finalized_start_end
+                keep_from = max(0, len(data) - _OVERLAP)
+                data = data[keep_from:]
+                buffer_start += keep_from
+            return _buffer_contains_direct_token(
+                data,
+                start_at=finalized_start - buffer_start,
+                finalized_start_end=len(data),
+            )
+    finally:
+        if fd is not None:
+            os.close(fd)
+
+
+def _regular_files(directory: Path) -> Iterator[Path]:
+    with os.scandir(directory) as listing:
+        entries = sorted(listing, key=lambda entry: entry.name)
+    for entry in entries:
+        if entry.is_symlink():
+            raise DirectTokenScanError
+        if entry.is_dir(follow_symlinks=False):
+            yield from _regular_files(Path(entry.path))
+        elif entry.is_file(follow_symlinks=False):
+            yield Path(entry.path)
+        else:
+            raise DirectTokenScanError
+
+
+def package_contains_direct_token(root: Path) -> bool:
+    """Return whether a package tree contains a supported direct issued-token form."""
+    try:
+        root_stat = os.stat(root, follow_symlinks=False)
+        if not stat.S_ISDIR(root_stat.st_mode):
+            raise DirectTokenScanError
+        return any(_file_contains_direct_token(path) for path in _regular_files(root))
+    except DirectTokenScanError:
+        raise
+    except OSError:
+        pass
+    raise DirectTokenScanError

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.68}/flash/server/domain/envs.py

@@ -16,6 +16,7 @@ import urllib.parse
 from pathlib import Path
 
 from flash.envs.loader import _github_token
+from flash.envs.package.direct_tokens import DirectTokenScanError, package_contains_direct_token
 from flash.envs.package.limits import (
     ARCHIVE_MEMBER_LIMIT,
     ARCHIVE_SCAN_MEMBER_LIMIT,
@@ -480,6 +481,14 @@ def publish_package(
     with tempfile.TemporaryDirectory(prefix="flash-env-publish-") as tmp:
         dest = Path(tmp)
         _safe_extract(tar_bytes, dest)
+        try:
+            contains_direct_token = package_contains_direct_token(dest)
+        except DirectTokenScanError:
+            raise EnvPublishError("env package could not be scanned safely") from None
+        if contains_direct_token:
+            raise EnvPublishError(
+                "env package contains a direct access token; remove it before publishing"
+            )
         return _github_publish(dest, name=name, key=key, project_slug=project_slug)
 
 

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.68}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.67"
+version = "1.2.68"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -137,7 +137,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.67"
+version = "1.2.68"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.68}/tests/test_env_publish.py

@@ -74,6 +74,93 @@ def _pkg_b64(files: dict[str, str]) -> str:
     return base64.b64encode(buf.getvalue()).decode()
 
 
+def _issued_token(prefix: str) -> str:
+    body_length = {"fslo_": 45, "hf_": 34, "pit_": 64}[prefix]
+    seed = (
+        "qR2_sT4-uV6wX8yZ0aB1cD3eF5gHjK7mN9pL"
+        if prefix == "fslo_"
+        else "qR2sT4uV6wX8yZ0aB1cD3eF5gHjK7mN9pL"
+    )
+    body = (seed * 2)[:body_length]
+    if prefix == "fslo_":
+        assert {"_", "-"} <= set(body)
+    return prefix + body
+
+
+def test_publish_rejects_direct_token_archive_before_github_publish(monkeypatch):
+    token = _issued_token("fslo_")
+    path = "scripts/bootstrap.sh"
+    package = {**_MINIMAL, path: f"#!/bin/sh\nexport FREESOLO_API_KEY={token}\n"}
+    calls: list[str] = []
+    monkeypatch.setattr(
+        envs,
+        "_github_publish",
+        lambda *_args, **_kwargs: calls.append("github") or pytest.fail("github must not start"),
+    )
+
+    with pytest.raises(envs.EnvPublishError) as excinfo:
+        envs.publish_package(
+            package_b64=_pkg_b64(package),
+            name="direct-bypass",
+            key={"org_slug": "acme"},
+            project_slug="checkout-bot",
+        )
+
+    assert excinfo.value.status == 400
+    error = str(excinfo.value)
+    assert "direct access token" in error
+    assert token not in error
+    assert path not in error
+    assert calls == []
+
+
+def test_publish_direct_token_scan_failure_is_safe_400(monkeypatch):
+    calls: list[str] = []
+    monkeypatch.setattr(
+        envs,
+        "package_contains_direct_token",
+        lambda _dest: (_ for _ in ()).throw(envs.DirectTokenScanError()),
+    )
+    monkeypatch.setattr(
+        envs,
+        "_github_publish",
+        lambda *_args, **_kwargs: calls.append("github") or pytest.fail("github must not start"),
+    )
+
+    with pytest.raises(envs.EnvPublishError) as excinfo:
+        envs.publish_package(
+            package_b64=_pkg_b64(_MINIMAL),
+            name="scan-failure",
+            key={"org_slug": "acme"},
+            project_slug="checkout-bot",
+        )
+
+    assert excinfo.value.status == 400
+    assert str(excinfo.value) == "env package could not be scanned safely"
+    assert calls == []
+
+
+def test_publish_direct_token_clean_control_reaches_github_publish(monkeypatch):
+    calls: list[str] = []
+
+    def fake_publish(dest, *, name, key, project_slug):
+        calls.append("github")
+        assert (dest / "environment.py").is_file()
+        return f"{key['org_slug']}/{project_slug}/{name}"
+
+    monkeypatch.setattr(envs, "_github_publish", fake_publish)
+
+    result = envs.publish_package(
+        package_b64=_pkg_b64({**_MINIMAL, "assets/data.bin": "clean opaque bytes"}),
+        name="clean-control",
+        key={"org_slug": "acme"},
+        project_slug="checkout-bot",
+    )
+
+    assert result == "acme/checkout-bot/clean-control"
+    assert calls == ["github"]
+
+
 def test_namespace_uses_org_slug():
     assert envs.namespace_for({"org_slug": "acme"}) == "acme"
     assert envs.namespace_for({"org": {"slug": "freesolo-co"}}) == "freesolo-co"

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.68}/tests/test_env_push.py

@@ -14,6 +14,11 @@ import pytest
 
 import flash.cli as cli
 from flash.cli.commands.env.push import _human_bytes, _UploadProgress
+from flash.envs.package.direct_tokens import (
+    _CHUNK_SIZE,
+    _OVERLAP,
+    package_contains_direct_token,
+)
 
 
 def _fake_client(capture: dict, *, slug: str = "acme/environment"):
@@ -51,6 +56,199 @@ def _args(
     return argparse.Namespace(path=str(path), name=name, project=project)
 
 
+def _token_body(prefix: str, length: int) -> str:
+    seed = (
+        "aB3_dE5-fG7hJ9kL2mN4pQ6rS8tUvW0xY1zC"
+        if prefix == "fslo_"
+        else "aB3dE5fG7hJ9kL2mN4pQ6rS8tUvW0xY1zC"
+    )
+    return (seed * 2)[:length]
+
+
+def _issued_token(prefix: str) -> str:
+    body_length = {"fslo_": 45, "hf_": 34, "pit_": 64}[prefix]
+    body = _token_body(prefix, body_length)
+    if prefix == "fslo_":
+        assert {"_", "-"} <= set(body)
+    return prefix + body
+
+
+def _deny_archive_and_upload(monkeypatch, calls: list[str]) -> None:
+    def deny_archive(_pkg):
+        calls.append("archive")
+        raise AssertionError("archive must not be created")
+
+    monkeypatch.setattr("flash.cli.commands.env.push._tar_b64", deny_archive)
+    monkeypatch.setattr(
+        "flash.client.client_from_config",
+        lambda: calls.append("upload") or pytest.fail("upload must not start"),
+    )
+
+
+@pytest.mark.parametrize(
+    ("prefix", "relative", "content"),
+    [
+        ("fslo_", "scripts/launch.sh", lambda token: f"#!/bin/sh\nexport API_KEY={token}\n"),
+        ("hf_", "helper.py", lambda token: f"TOKEN = {token!r}\n"),
+        ("pit_", "environment.py", lambda token: f"TOKEN = {token!r}\n"),
+    ],
+    ids=["freesolo-shell", "hugging-face-python", "prime-entrypoint"],
+)
+def test_push_rejects_direct_tokens_before_archive_or_upload(
+    monkeypatch, tmp_path, capsys, prefix, relative, content
+):
+    env_dir = tmp_path / "env"
+    env_dir.mkdir()
+    (env_dir / "environment.py").write_text("def load_environment(**k):\n    return None\n")
+    target = env_dir / relative
+    target.parent.mkdir(parents=True, exist_ok=True)
+    token = _issued_token(prefix)
+    target.write_text(content(token))
+    calls: list[str] = []
+    _deny_archive_and_upload(monkeypatch, calls)
+
+    assert cli.cmd_env_push(_args(env_dir)) == 1
+
+    error = capsys.readouterr().err
+    assert "direct access token" in error
+    assert token not in error
+    assert relative not in error
+    assert calls == []
+
+
+@pytest.mark.parametrize(
+    ("prefix", "body_length"),
+    [("fslo_", 45), ("hf_", 34), ("pit_", 64)],
+    ids=["freesolo", "hugging-face", "prime"],
+)
+@pytest.mark.parametrize("delta", [-1, 1], ids=["minus-one", "plus-one"])
+def test_direct_token_neighbor_body_lengths_are_clean(tmp_path, prefix, body_length, delta):
+    package = tmp_path / "package"
+    package.mkdir()
+    value = prefix + _token_body(prefix, body_length + delta)
+    (package / "data.bin").write_bytes(b" " + value.encode() + b" ")
+
+    assert package_contains_direct_token(package) is False
+
+
+@pytest.mark.parametrize(
+    "token_start",
+    [_CHUNK_SIZE - 3, _CHUNK_SIZE - _OVERLAP - 3],
+    ids=["read-boundary", "overlap-cutoff"],
+)
+def test_push_rejects_direct_token_across_chunk_boundary(
+    monkeypatch, tmp_path, capsys, token_start
+):
+    env_dir = tmp_path / "env"
+    env_dir.mkdir()
+    (env_dir / "environment.py").write_text("def load_environment(**k):\n    return None\n")
+    token = _issued_token("hf_").encode()
+    (env_dir / "opaque.bin").write_bytes(b" " * token_start + token + b"\x00" * 256)
+    calls: list[str] = []
+    _deny_archive_and_upload(monkeypatch, calls)
+
+    assert cli.cmd_env_push(_args(env_dir)) == 1
+
+    error = capsys.readouterr().err
+    assert "direct access token" in error
+    assert token.decode() not in error
+    assert "opaque.bin" not in error
+    assert calls == []
+
+
+def test_push_allows_clean_binary_placeholders_and_embedded_direct_token_shapes(
+    monkeypatch, tmp_path
+):
+    env_dir = tmp_path / "env"
+    env_dir.mkdir()
+    embedded = _issued_token("pit_")
+    placeholders = [
+        "fslo_" + "your_api_token_here".ljust(45, "_"),
+        "hf_" + "x" * 34,
+        "pit_" + "0" * 64,
+    ]
+    (env_dir / "environment.py").write_text(
+        "def load_environment(**k):\n    return None\n" + "\n".join(placeholders)
+    )
+    (env_dir / "notes.txt").write_text(
+        "ordinary text without credentials\n"
+        "hf_resume_checkpoint\n"
+        "fslo_retry_after_close\n"
+        "pit_environment_identifier\n"
+    )
+    (env_dir / "opaque.bin").write_bytes(b"\x00\xffclean\x80binary\x00")
+    (env_dir / "embedded-left.bin").write_bytes(("left" + embedded + " ").encode())
+    (env_dir / "overlong.bin").write_bytes((" pit_" + "aB3" * 22 + " ").encode())
+    cap: dict = {}
+    monkeypatch.setattr("flash.client.client_from_config", _fake_client(cap))
+
+    assert cli.cmd_env_push(_args(env_dir)) == 0
+    assert cap["package_b64"]
+
+
+def test_push_allows_direct_token_shape_with_invalid_left_at_retained_boundary(
+    monkeypatch, tmp_path
+):
+    env_dir = tmp_path / "env"
+    env_dir.mkdir()
+    (env_dir / "environment.py").write_text("def load_environment(**k):\n    return None\n")
+    token = _issued_token("pit_").encode()
+    prefix_padding = b" " * (_CHUNK_SIZE - _OVERLAP - 1)
+    payload = prefix_padding + b"z" + token + b"\x00" + b"next chunk"
+    (env_dir / "retained-boundary.bin").write_bytes(payload)
+    cap: dict = {}
+    monkeypatch.setattr("flash.client.client_from_config", _fake_client(cap))
+
+    assert cli.cmd_env_push(_args(env_dir)) == 0
+    assert cap["package_b64"]
+
+
+def test_direct_token_scan_error_drops_oserror_details(tmp_path):
+    from flash.envs.package.direct_tokens import DirectTokenScanError, package_contains_direct_token
+
+    missing = tmp_path / "sensitive-package-path"
+    with pytest.raises(DirectTokenScanError) as excinfo:
+        package_contains_direct_token(missing)
+
+    assert str(excinfo.value) == "package scan failed"
+    assert excinfo.value.__context__ is None
+    assert str(missing) not in str(excinfo.value)
+
+
+def test_push_fails_closed_when_direct_token_scan_sees_unexpected_member(
+    monkeypatch, tmp_path, capsys
+):
+    from flash.cli.commands.env import push as envpush
+
+    env_dir = tmp_path / "env"
+    env_dir.mkdir()
+    (env_dir / "environment.py").write_text("def load_environment(**k):\n    return None\n")
+    outside = tmp_path / "outside"
+    outside.write_text("clean\n")
+    real_copy = envpush._copy_env_sidecars
+
+    def copy_with_link(env_root, dest, *, entrypoint, include_full_tree):
+        real_copy(
+            env_root,
+            dest,
+            entrypoint=entrypoint,
+            include_full_tree=include_full_tree,
+        )
+        (dest / "unexpected-link").symlink_to(outside)
+
+    monkeypatch.setattr(envpush, "_copy_env_sidecars", copy_with_link)
+    calls: list[str] = []
+    _deny_archive_and_upload(monkeypatch, calls)
+
+    assert cli.cmd_env_push(_args(env_dir)) == 1
+
+    error = capsys.readouterr().err
+    assert "could not be scanned safely" in error
+    assert "unexpected-link" not in error
+    assert str(outside) not in error
+    assert calls == []
+
+
 def test_push_single_py_module_is_packaged(monkeypatch, tmp_path, capsys):
     env_file = tmp_path / "environment.py"
     env_file.write_text("def load_environment(**k):\n    return None\n")

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.68}/uv.lock

@@ -1052,7 +1052,7 @@ wheels = [
 
 [[package]]
 name = "freesolo-flash"
-version = "1.2.67"
+version = "1.2.68"
 source = { editable = "." }
 
 [package.optional-dependencies]