benchmax 0.2.1__tar.gz → 0.2.2__tar.gz

{benchmax-0.2.1 → benchmax-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchmax
-Version: 0.2.1
+Version: 0.2.2
 Summary: Platform-independent runtime for grouped LLM environments
 Author: benchmax Authors
 Classifier: Operating System :: OS Independent
@@ -98,6 +98,8 @@ environment class + constructor arguments
 
 benchmax creates the portable artifact so the same environment can be loaded outside the author's checkout. `castform` handles uploading the bundle, validating it remotely, and using it for training.
 
+Remote runtimes install `pip_dependencies` without enabling prereleases globally. If a dependency graph needs a prerelease, list that package explicitly even when it would normally be transitive. For example, use `pip_dependencies=["parent-package==1.0.0", "transitive-package==2.0.0rc1"]`.
+
 ## further reading
 
 - [base environment guide](src/benchmax/envs/base/README.md)

{benchmax-0.2.1 → benchmax-0.2.2}/README.md

@@ -83,6 +83,8 @@ environment class + constructor arguments
 
 benchmax creates the portable artifact so the same environment can be loaded outside the author's checkout. `castform` handles uploading the bundle, validating it remotely, and using it for training.
 
+Remote runtimes install `pip_dependencies` without enabling prereleases globally. If a dependency graph needs a prerelease, list that package explicitly even when it would normally be transitive. For example, use `pip_dependencies=["parent-package==1.0.0", "transitive-package==2.0.0rc1"]`.
+
 ## further reading
 
 - [base environment guide](src/benchmax/envs/base/README.md)

{benchmax-0.2.1 → benchmax-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchmax"
-version = "0.2.1"
+version = "0.2.2"
 description = "Platform-independent runtime for grouped LLM environments"
 readme = "README.md"
 authors = [{ name = "benchmax Authors" }]

{benchmax-0.2.1 → benchmax-0.2.2}/src/benchmax/auth.py

@@ -16,6 +16,7 @@ from dataclasses import dataclass, field
 from typing import Protocol, runtime_checkable
 
 import httpx
+from openai import AsyncOpenAI, OpenAI
 
 __all__ = [
     "InjectedAuth",
@@ -24,6 +25,8 @@ __all__ = [
     "RequestModelAuth",
     "StaticBearerAuth",
     "bind_model_auth",
+    "create_async_openai_client",
+    "create_openai_client",
 ]
 
 
@@ -99,6 +102,52 @@ class RequestModelAuth(httpx.Auth):
         yield request
 
 
+def create_openai_client(
+    *,
+    model: str,
+    base_url: str,
+    auth: ModelAuth,
+    request_id: str,
+    max_retries: int = 2,
+) -> OpenAI:
+    """Create a synchronous OpenAI-compatible client with explicit auth."""
+
+    context = ModelRequestContext(
+        base_url=base_url,
+        model=model,
+        rollout_id=request_id,
+    )
+    return OpenAI(
+        base_url=base_url,
+        api_key="benchmax-explicit-auth",
+        http_client=httpx.Client(auth=RequestModelAuth(auth, context)),
+        max_retries=max_retries,
+    )
+
+
+def create_async_openai_client(
+    *,
+    model: str,
+    base_url: str,
+    auth: ModelAuth,
+    request_id: str,
+    max_retries: int = 2,
+) -> AsyncOpenAI:
+    """Create an asynchronous OpenAI-compatible client with explicit auth."""
+
+    context = ModelRequestContext(
+        base_url=base_url,
+        model=model,
+        rollout_id=request_id,
+    )
+    return AsyncOpenAI(
+        base_url=base_url,
+        api_key="benchmax-explicit-auth",
+        http_client=httpx.AsyncClient(auth=RequestModelAuth(auth, context)),
+        max_retries=max_retries,
+    )
+
+
 def _resolve_headers_sync(
     auth: ModelAuth,
     context: ModelRequestContext,

{benchmax-0.2.1 → benchmax-0.2.2}/src/benchmax/bundle.py

@@ -203,6 +203,9 @@ def dump_bundle(
         env_class: A concrete Environment implementation.
         constructor_args: kwargs for ``env_class(**...)`` on load.
         pip_dependencies: Recorded in metadata. NOT installed by this call.
+            List every required prerelease explicitly, including a prerelease
+            that would otherwise only be a transitive dependency. Remote
+            runtimes do not enable prereleases globally.
         local_modules: Additional modules to pickle by value. Use this for
             local source outside the environment's own Python project. Such
             source otherwise fails loudly unless its installed distribution is

{benchmax-0.2.1 → benchmax-0.2.2}/src/benchmax/envs/harbor/README.md

@@ -50,6 +50,8 @@ for each rollout, `HarborEnv` gives the task to the configured agent and sandbox
 
 the rollout request supplies the model endpoint and credential used by the agent. verifier and judge configuration remains separate. because agents usually run inside remote sandboxes, `HarborEnv` requests a publicly reachable model endpoint by default.
 
+configure the agent harness with compaction and summarization disabled: training assumes a linear chat history where each model call extends the previous messages. a harness that rewrites its context mid-rollout produces a transcript that cannot be trained on.
+
 benchmax owns rollout-group concurrency. `max_concurrent_trials` can additionally cap how many Harbor sandbox trials this environment runs at once.
 
 ## sandboxes and credentials
@@ -58,7 +60,7 @@ the Harbor environment configuration selects the sandbox provider. credentials a
 
 Modal and Daytona currently use explicit `sandbox_credentials`. these values become part of the environment bundle, so use dedicated, revocable credentials rather than personal ones.
 
-Harbor verifiers currently receive credentials as static environment variables in `TrialVerifierConfig`. runtime credential injection is available to the rollout agent, not to the verifier.
+Harbor verifiers receive credentials as static environment variables in `TrialVerifierConfig`. the only values injected at rollout time are the model endpoint and API key the agent uses; nothing is injected into the verifier.
 
 ## bundled custom agents
 

benchmax-0.2.2/src/benchmax/rag/__init__.py

@@ -0,0 +1,13 @@
+"""Runtime building blocks for retrieval-augmented Benchmax environments."""
+
+from benchmax.rag.embed import DEFAULT_EMBED_MODEL, OpenAIEmbedder
+from benchmax.rag.env import RagEnv
+from benchmax.rag.search import SearchClient, SearchResult
+
+__all__ = [
+    "DEFAULT_EMBED_MODEL",
+    "OpenAIEmbedder",
+    "SearchClient",
+    "RagEnv",
+    "SearchResult",
+]

benchmax-0.2.2/src/benchmax/rag/embed.py

@@ -0,0 +1,79 @@
+"""OpenAI-compatible query embedding for RAG search clients.
+
+The rollout-facing provider search clients (Turbopuffer / Pinecone / Chroma)
+accept an optional async ``embed_fn``. Wiring one in makes vector / hybrid
+retrieval work regardless of how the user's index was built — turbopuffer
+vector/hybrid (no server-side embed), a pinecone index NOT on the hosted model,
+or a non-cloud chroma collection.
+
+``qa-gen`` does NOT need this (it reads chunks directly); it's only for retrieval.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from benchmax.auth import ModelAuth, create_async_openai_client
+
+# The served, non-hidden embeddings model in the llm-proxy catalog.
+DEFAULT_EMBED_MODEL = "text-embedding-3-large"
+
+
+@dataclass(frozen=True, slots=True)
+class OpenAIEmbedder:
+    """Pickle-safe embedding callable with explicit call-time model auth.
+
+    ``auth`` follows the same contract as :class:`benchmax.rewards.Judge`.
+    Managed environments use ``InjectedAuth("embedding")``; customer-owned
+    endpoints use ``StaticBearerAuth``. No platform or SDK environment
+    credential is inferred.
+    """
+
+    model: str
+    base_url: str
+    auth: ModelAuth
+    timeout: float | None = 60.0
+    max_retries: int = 2
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.auth, ModelAuth):
+            raise TypeError("embedding auth must implement ModelAuth")
+        if not isinstance(self.model, str) or not self.model.strip():
+            raise ValueError("embedding model must be non-empty")
+        if not isinstance(self.base_url, str) or not self.base_url.strip():
+            raise ValueError("embedding base_url must be non-empty")
+        if self.timeout is not None and (
+            isinstance(self.timeout, bool)
+            or not isinstance(self.timeout, (int, float))
+            or self.timeout <= 0
+        ):
+            raise ValueError("embedding timeout must be positive or None")
+        if (
+            isinstance(self.max_retries, bool)
+            or not isinstance(self.max_retries, int)
+            or self.max_retries < 0
+        ):
+            raise ValueError("embedding max_retries must be non-negative")
+
+    async def __call__(self, texts: list[str]) -> list[list[float]]:
+        if not isinstance(texts, list) or any(not isinstance(text, str) for text in texts):
+            raise TypeError("embedding input must be a list of strings")
+        if not texts:
+            return []
+
+        client = create_async_openai_client(
+            model=self.model,
+            base_url=self.base_url,
+            auth=self.auth,
+            request_id="rag-embedding",
+            max_retries=self.max_retries,
+        )
+        try:
+            response = await client.embeddings.create(
+                model=self.model,
+                input=texts,
+                timeout=self.timeout,
+            )
+            return [item.embedding for item in response.data]
+        finally:
+            await client.close()