benchmax 0.1.2.dev23__tar.gz → 0.1.2.dev25__tar.gz

{benchmax-0.1.2.dev23 → benchmax-0.1.2.dev25}/PKG-INFO

@@ -1,18 +1,21 @@
 Metadata-Version: 2.4
 Name: benchmax
-Version: 0.1.2.dev23
+Version: 0.1.2.dev25
 Summary: Framework-Agnostic RL Environments for LLM Fine-Tuning
-Author: cgft.io
+Author: castie@castform.com
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.12
+Requires-Python: ==3.12.*
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp>=3.13.1
 Requires-Dist: asyncio>=4.0.0
 Requires-Dist: cloudpickle>=3.0.0
 Requires-Dist: datasets>=4.0.0
-Requires-Dist: expt-logger>=0.1.0.dev22
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: json-repair>=0.59.10
+Requires-Dist: openai>=2.15.0
+Requires-Dist: pydantic>=2.0.0
 Provides-Extra: mcp
 Requires-Dist: fastmcp~=2.12.0; extra == "mcp"
 Requires-Dist: pyjwt>=2.10.1; extra == "mcp"
@@ -20,6 +23,29 @@ Provides-Extra: skypilot
 Requires-Dist: skypilot[aws,gcp]~=0.8.1; extra == "skypilot"
 Requires-Dist: pip>=25.3; extra == "skypilot"
 Requires-Dist: msrestazure>=0.6.4.post1; extra == "skypilot"
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1.5; extra == "excel"
+Provides-Extra: excel-mac-windows
+Requires-Dist: openpyxl>=3.1.5; extra == "excel-mac-windows"
+Requires-Dist: xlwings>=0.33.16; extra == "excel-mac-windows"
+Provides-Extra: crm
+Requires-Dist: python-dateutil>=2.9.0.post0; extra == "crm"
+Provides-Extra: rag
+Requires-Dist: keybert>=0.8; extra == "rag"
+Requires-Dist: langchain-text-splitters>=0.3.0; extra == "rag"
+Requires-Dist: nest-asyncio>=1.5.0; extra == "rag"
+Requires-Dist: ragas>=0.4.3; extra == "rag"
+Requires-Dist: ruamel-yaml>=0.19.1; extra == "rag"
+Requires-Dist: scikit-learn>=1.8.0; extra == "rag"
+Requires-Dist: sentence-transformers>=5.2.3; extra == "rag"
+Requires-Dist: tqdm>=4.66.0; extra == "rag"
+Provides-Extra: traces
+Provides-Extra: chroma
+Requires-Dist: chromadb>=1.0.0; extra == "chroma"
+Provides-Extra: pinecone
+Requires-Dist: pinecone>=5.0.0; extra == "pinecone"
+Provides-Extra: turbopuffer
+Requires-Dist: turbopuffer>=1.16.2; extra == "turbopuffer"
 Dynamic: license-file
 
 <picture>
@@ -31,57 +57,26 @@ Dynamic: license-file
 <div align="center">
 </div>
 <div id="badges" align="center">
-  <a href="https://cgft.io">
-    <img src="https://img.shields.io/badge/cgft.io-blue?style=for-the-badge" alt="Website"/>
-  </a>
-  <a href="https://x.com/cgftlabs">
-    <img src="https://img.shields.io/badge/Follow @cgftlabs-black?style=for-the-badge&logo=X&logoColor=white" alt="@cgftlabs"/>
+  <a href="https://castform.com">
+    <img src="https://img.shields.io/badge/castform.com-blue?style=for-the-badge" alt="Website"/>
   </a>
 </div>
 <div align="center" style="line-height: 1;">
-  <a href="https://github.com/girishbarca/benchmax/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-Apache_2.0-blue.svg"/></a>
+  <a href="./LICENSE"><img alt="License" src="https://img.shields.io/badge/License-Apache_2.0-blue.svg"/></a>
 </div>
 
 ## 📌 News
 
 - **[29 Oct 2025]** 🎉 Added support for easy multi-node parallelization across all major cloud providers using [SkyPilot](https://github.com/skypilot-org/skypilot)
-- **[29 Oct 2025]** 🎉 Integration with [SkyRL](https://github.com/NovaSky-AI/SkyRL) for distributed RL training across clusters
-- **[Upcoming]** 🛠️ Integration with Tinker API.
-
-## 📘 Quickstart
-
-**Example: Multi-node parallelization of Excel Env with SkyRL and SkyPilot**
-
-RL environments can be computationally expensive to run (e.g. running tests). To handle these workloads efficiently, we distribute rollouts across multiple nodes using **SkyPilot**, horizontally scaling `benchmax` across cloud providers like GCP, AWS, Azure, etc.
-
-**SkyRL** is a training framework `benchmax` is currently integrated with. Use our ***SkyRL*** integration to RL finetune Qwen-2.5 to do spreadsheet manipulation using a excel MCP parallelized across multiple nodes. The environment is defined in [`benchmax.envs.excel.excel_env.ExcelEnvSkypilot`](/src/benchmax/envs/excel/excel_env.py)
-
-1. **Prepare the dataset**
-    
-    ```bash
-    uv run src/benchmax/adapters/skyrl/benchmax_data_process.py \
-      --local_dir ~/data/excel \
-      --dataset_name spreadsheetbench \
-      --env_path benchmax.envs.excel.excel_env.ExcelEnvLocal
-    ```
-
-    Note: We are using `ExcelEnvLocal` instead of `ExcelEnvSkypilot` because the MCP is only used for listing tools to prepare the system prompt.
-    
-2. **Run training and parallelize Excel environment**
-    
-    ```bash
-    bash examples/skyrl/run_benchmax_excel.sh
-    ```
-
-This excel env example will spin up 5 nodes with 20 servers per node (total 100 MCP server in parallel). For more details, check out [multi-node parallelization](/src/benchmax/envs/mcp/README.md) and [SkyRL integration](/examples/skyrl/README.md).
 
 ## ℹ️ Overview
 
 `benchmax` comes with:
 
 - A collection of ready-to-use reinforcement learning (RL) environments for LLM fine-tuning ranging from multi-hop search to spreadsheet manipulation to CRM agents
-- An easy to define, compose, and parallelize your own environments, including leveraging the existing ecosystem of MCP servers
-- Built-in integrations with popular RL training libraries (skyrl, etc.). `benchmax` is trainer-agnostic by design
+- An easy way to define, compose, and parallelize your own environments, including leveraging the existing ecosystem of MCP servers
+- Trainer-agnostic by design — `BaseEnv` exposes a small async interface (`list_tools`, `run_tool`, `compute_reward`, plus optional rollout lifecycle hooks) that any rollout loop can drive
+- Optional batteries-included add-ons: synthetic RAG dataset generation (`benchmax[rag]`), agent trace import (`benchmax[traces]`), and clients for the Castform training platform (`benchmax.platform`)
 
 Define your environment as:
 
@@ -93,26 +88,22 @@ Rollout management, parallel execution, etc. comes out of the box.
 
 ⭐ Star our repository to show your support!
 
-## 💡 Core Features
+## 💡 Core Features
 
 **Built-in examples & templates**
 
 Get started with ready to use recipes, from Wikipedia search to spreadsheet manipulation. Easy to copy, customize, and extend. And yes, more are on the way.
 
-**Trainer integrations**
-
-Use your own trainer or training framework - no lock-in. `benchmax` is already integrated into SkyRL, with more integrations (Tinker, etc.) coming soon!
-
 **MCP support**
 
 Tap into the growing MCP ecosystem and integrate them as tools within your environments.
 
 **Multi-node parallel execution**
 
-Multi-node parallelization enabled out of the box with state isolation across roll-outs (e.g. editing files on filesystem, etc.).  
+Multi-node parallelization enabled out of the box with state isolation across roll-outs (e.g. editing files on filesystem, etc.).
 
 
-## 🌐 Creating & Training with Environments
+## 🌐 Creating Environments
 
 ### What is an environment?
 
@@ -128,9 +119,10 @@ We also support MCP servers natively, allowing you to easily leverage the many s
 Ready-to-use environments with pre-configured tools and reward functions.
 
 - [CRM](/src/benchmax/envs/crm/README.md)
-- [Excel](/src/benchmax/envs/excel/README.md) 
+- [Excel](/src/benchmax/envs/excel/README.md)
 - [Math](/src/benchmax/envs/math/README.md)
 - [Wikipedia](/src/benchmax/envs/wikipedia/README.md)
+- [PostgreSQL search](/src/benchmax/envs/postgres_search/) (`benchmax[rag]`)
 
 ### How do I create a custom environment?
 
@@ -142,12 +134,6 @@ Ready-to-use environments with pre-configured tools and reward functions.
 
 - Check out our excel spreadsheet RL environment: `benchmax.envs.excel.excel_env.ExcelEnv`
 
-### How do I use an environment with my preferred RL Trainer?
-
-We currently have integrations with SkyRL. More incoming!
-
-[`benchmax` environments with skyrl](/examples/skyrl/README.md)
-
 ### I want a specific environment
 
 Open an issue and tag us & we will look into building you one!
@@ -157,36 +143,46 @@ Open an issue and tag us & we will look into building you one!
 ## 🎯 Motivation
 
 - **Modularity and Simplicity**:
-    
+
     We set out to build a lightweight, modular system for defining RL environments—breaking them down into simple, composable parts: tools, tool output parsing, and reward functions.
-    
-    The goal’s to make it easy for software engineers to build and experiment with RL environments without needing deep RL expertise.
-    
-- **Trainer Integrations**:
-    
-    There’s been lots of new RL training frameworks popping up (e.g., numerous forks of verl) & we expect this to continue. They are often tightly coupled with specific environments, leading to fragmentation and limited compatibility. 
-    
-    We are building `benchmax` as a standalone library with integrations to these different training frameworks & as an easy way for new frameworks to tap into an existing pool of environments. We're already integrated with SkyRL (Tinker coming soon)!
-    
+
+    The goal's to make it easy for software engineers to build and experiment with RL environments without needing deep RL expertise.
+
 - **Task Recipes and Ideas**:
-    
+
     We want `benchmax` to be a living library of reusable, RL-compatible task recipes, ready to inspire and extend beyond the usual suspects like math and coding. We aim to support more real-world workflows, including open-ended and long-horizon tasks.
-    
+
 - **Parallelization and Cloud Compatibility**:
     - Enable efficient parallelization with maintained statefulness between rollouts.
     - Facilitate easy deployment and scalability in cloud environments.
 
 - **MCP as a first class citizen**:
-    
-    There has been an explosion of MCP servers/tools built out for use-cases ranging from browser use to excel to game creation.`benchmax` allows folks to leverage and compose these existing MCP servers to build environments integrated with real world systems e.g. excel
-    
+
+    There has been an explosion of MCP servers/tools built out for use-cases ranging from browser use to excel to game creation. `benchmax` allows folks to leverage and compose these existing MCP servers to build environments integrated with real world systems e.g. excel
+
 
 ## 🤝 Contributing
 
-We welcome new environment recipes, bug reports, and trainer integrations!
+We welcome new environment recipes and bug reports!
 
 ⭐ Star our repository to show your support!
 
+## 📦 Add-ons
+
+In addition to the core env library, `benchmax` ships several optional
+modules behind extras:
+
+| Extra | Module | Purpose |
+|---|---|---|
+| `benchmax[rag]` | `benchmax.rag.*` | Markdown chunking, corpus indexing (Postgres / Chroma / Pinecone / Turbopuffer), synthetic QA dataset generation, RAG-specific reward rubrics |
+| `benchmax[traces]` | `benchmax.traces` | Agentic trace import (Braintrust today, Langfuse coming) and provider-agnostic processing pipeline |
+| `benchmax[chroma]` / `[pinecone]` / `[turbopuffer]` | `benchmax.rag.corpus.*` | Corpus-backend pins (combine with `[rag]`) |
+| _(core)_ | `benchmax.platform` | HTTP clients for the Castform platform — storage uploads, training-job launch, rollout server. Used both internally by `benchmax.rag` and by the high-level [`castform-sdk`](https://pypi.org/project/castform-sdk/). |
+
+All platform URLs derive from `CASTFORM_BASE_DOMAIN` (default
+`castform.com`) with per-component overrides; see
+[`benchmax.config`](src/benchmax/config.py).
+
 ## 📜 License
 
-Apache 2.0 © 2025 CGFT Inc.
+Apache 2.0 © 2026 Castform

benchmax-0.1.2.dev25/README.md

@@ -0,0 +1,138 @@
+<picture>
+  <img alt="Benchmax" src="./static/benchmax.png"  width="full">
+</picture>
+
+## benchmax: Framework-Agnostic RL Environments for LLM Fine-Tuning
+*A lightweight, training-framework agnostic library for defining, running, and parallelizing environments, to fine-tune OSS LLMs with reinforcement learning.*
+<div align="center">
+</div>
+<div id="badges" align="center">
+  <a href="https://castform.com">
+    <img src="https://img.shields.io/badge/castform.com-blue?style=for-the-badge" alt="Website"/>
+  </a>
+</div>
+<div align="center" style="line-height: 1;">
+  <a href="./LICENSE"><img alt="License" src="https://img.shields.io/badge/License-Apache_2.0-blue.svg"/></a>
+</div>
+
+## 📌 News
+
+- **[29 Oct 2025]** 🎉 Added support for easy multi-node parallelization across all major cloud providers using [SkyPilot](https://github.com/skypilot-org/skypilot)
+
+## ℹ️ Overview
+
+`benchmax` comes with:
+
+- A collection of ready-to-use reinforcement learning (RL) environments for LLM fine-tuning ranging from multi-hop search to spreadsheet manipulation to CRM agents
+- An easy way to define, compose, and parallelize your own environments, including leveraging the existing ecosystem of MCP servers
+- Trainer-agnostic by design — `BaseEnv` exposes a small async interface (`list_tools`, `run_tool`, `compute_reward`, plus optional rollout lifecycle hooks) that any rollout loop can drive
+- Optional batteries-included add-ons: synthetic RAG dataset generation (`benchmax[rag]`), agent trace import (`benchmax[traces]`), and clients for the Castform training platform (`benchmax.platform`)
+
+Define your environment as:
+
+1. A **toolset** (LLM calls, external APIs, calculators, MCPs, etc.).
+2. **Output parsing** logic to extract structured observations.
+3. **Reward functions** to score model outputs.
+
+Rollout management, parallel execution, etc. comes out of the box.
+
+⭐ Star our repository to show your support!
+
+## 💡 Core Features
+
+**Built-in examples & templates**
+
+Get started with ready to use recipes, from Wikipedia search to spreadsheet manipulation. Easy to copy, customize, and extend. And yes, more are on the way.
+
+**MCP support**
+
+Tap into the growing MCP ecosystem and integrate them as tools within your environments.
+
+**Multi-node parallel execution**
+
+Multi-node parallelization enabled out of the box with state isolation across roll-outs (e.g. editing files on filesystem, etc.).
+
+
+## 🌐 Creating Environments
+
+### What is an environment?
+
+An environment consists of:
+
+- A list of tools that an LLM can call
+- A list of reward functions that evaluate the quality & correctness of the model's final output.
+
+We also support MCP servers natively, allowing you to easily leverage the many servers built by the community.
+
+### Pre-built environments
+
+Ready-to-use environments with pre-configured tools and reward functions.
+
+- [CRM](/src/benchmax/envs/crm/README.md)
+- [Excel](/src/benchmax/envs/excel/README.md)
+- [Math](/src/benchmax/envs/math/README.md)
+- [Wikipedia](/src/benchmax/envs/wikipedia/README.md)
+- [PostgreSQL search](/src/benchmax/envs/postgres_search/) (`benchmax[rag]`)
+
+### How do I create a custom environment?
+
+1. [With existing MCP servers](/src/benchmax/envs/mcp/README.md) (Built-in support for multi-node parallelization)
+
+2. [Extend BaseEnv](/src/benchmax/envs/README.md)
+
+### How about more complex environments?
+
+- Check out our excel spreadsheet RL environment: `benchmax.envs.excel.excel_env.ExcelEnv`
+
+### I want a specific environment
+
+Open an issue and tag us & we will look into building you one!
+
+---
+
+## 🎯 Motivation
+
+- **Modularity and Simplicity**:
+
+    We set out to build a lightweight, modular system for defining RL environments—breaking them down into simple, composable parts: tools, tool output parsing, and reward functions.
+
+    The goal's to make it easy for software engineers to build and experiment with RL environments without needing deep RL expertise.
+
+- **Task Recipes and Ideas**:
+
+    We want `benchmax` to be a living library of reusable, RL-compatible task recipes, ready to inspire and extend beyond the usual suspects like math and coding. We aim to support more real-world workflows, including open-ended and long-horizon tasks.
+
+- **Parallelization and Cloud Compatibility**:
+    - Enable efficient parallelization with maintained statefulness between rollouts.
+    - Facilitate easy deployment and scalability in cloud environments.
+
+- **MCP as a first class citizen**:
+
+    There has been an explosion of MCP servers/tools built out for use-cases ranging from browser use to excel to game creation. `benchmax` allows folks to leverage and compose these existing MCP servers to build environments integrated with real world systems e.g. excel
+
+
+## 🤝 Contributing
+
+We welcome new environment recipes and bug reports!
+
+⭐ Star our repository to show your support!
+
+## 📦 Add-ons
+
+In addition to the core env library, `benchmax` ships several optional
+modules behind extras:
+
+| Extra | Module | Purpose |
+|---|---|---|
+| `benchmax[rag]` | `benchmax.rag.*` | Markdown chunking, corpus indexing (Postgres / Chroma / Pinecone / Turbopuffer), synthetic QA dataset generation, RAG-specific reward rubrics |
+| `benchmax[traces]` | `benchmax.traces` | Agentic trace import (Braintrust today, Langfuse coming) and provider-agnostic processing pipeline |
+| `benchmax[chroma]` / `[pinecone]` / `[turbopuffer]` | `benchmax.rag.corpus.*` | Corpus-backend pins (combine with `[rag]`) |
+| _(core)_ | `benchmax.platform` | HTTP clients for the Castform platform — storage uploads, training-job launch, rollout server. Used both internally by `benchmax.rag` and by the high-level [`castform-sdk`](https://pypi.org/project/castform-sdk/). |
+
+All platform URLs derive from `CASTFORM_BASE_DOMAIN` (default
+`castform.com`) with per-component overrides; see
+[`benchmax.config`](src/benchmax/config.py).
+
+## 📜 License
+
+Apache 2.0 © 2026 Castform

{benchmax-0.1.2.dev23 → benchmax-0.1.2.dev25}/pyproject.toml

@@ -1,16 +1,19 @@
 [project]
 name = "benchmax"
-version = "0.1.2.dev23"
+version = "0.1.2.dev25"
 description = "Framework-Agnostic RL Environments for LLM Fine-Tuning"
 readme = "README.md"
-authors = [{ name = "cgft.io" }]
-requires-python = ">=3.12"
+authors = [{ name = "castie@castform.com" }]
+requires-python = "==3.12.*"
 dependencies = [
     "aiohttp>=3.13.1",
     "asyncio>=4.0.0",
     "cloudpickle>=3.0.0",
     "datasets>=4.0.0",
-    "expt-logger>=0.1.0.dev22",
+    "httpx>=0.27.0",
+    "json-repair>=0.59.10",
+    "openai>=2.15.0",
+    "pydantic>=2.0.0",
 ]
 classifiers = [
     "Programming Language :: Python :: 3",
@@ -34,6 +37,23 @@ skypilot = [
     "pip>=25.3",
     "msrestazure>=0.6.4.post1",
 ]
+excel = ["openpyxl>=3.1.5"]
+excel-mac-windows = ["openpyxl>=3.1.5", "xlwings>=0.33.16"]
+crm = ["python-dateutil>=2.9.0.post0"]
+rag = [
+    "keybert>=0.8",
+    "langchain-text-splitters>=0.3.0",
+    "nest-asyncio>=1.5.0",
+    "ragas>=0.4.3",
+    "ruamel-yaml>=0.19.1",
+    "scikit-learn>=1.8.0",
+    "sentence-transformers>=5.2.3",
+    "tqdm>=4.66.0",
+]
+traces = []
+chroma = ["chromadb>=1.0.0"]
+pinecone = ["pinecone>=5.0.0"]
+turbopuffer = ["turbopuffer>=1.16.2"]
 
 [dependency-groups]
 dev = [
@@ -51,23 +71,6 @@ skypilot = [
     "pip>=25.3",                      # Added as needed for skypilot launch
     "msrestazure>=0.6.4.post1",
 ]
-skyrl = [
-    "grpcio>=1.60.0",
-    "hydra-core>=1.3.2",
-    "omegaconf>=2.3.0",
-    "ray>=2.48.0",
-    "skyrl-gym>=0.1.1",
-    "skyrl-train[vllm]>=0.2.0",
-]
 excel = ["openpyxl>=3.1.5"]
 excel-mac-windows = ["openpyxl>=3.1.5", "xlwings>=0.33.16"]
-crm = ["python-dateutil>=2.9.0.post0", "simple-salesforce>=1.12.9"]
-
-[tool.uv]
-conflicts = [[{ group = "skypilot" }, { group = "skyrl" }]]
-
-# [tool.uv.extra-build-dependencies]
-# flash-attn = [{ requirement = "torch", match-runtime = true }]
-
-# [tool.uv.extra-build-variables]
-# flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE" }
+crm = ["python-dateutil>=2.9.0.post0"]