flyai-bench 0.1.0__tar.gz

flyai_bench-0.1.0/.github/workflows/validate.yml

@@ -0,0 +1,53 @@
+name: Validate Submission
+
+on:
+  pull_request:
+    paths:
+      - 'experiments/evaluation/**'
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Validate new submissions
+        run: python validate_submission.py check-pr
+
+  update-leaderboard:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    needs: validate
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Rebuild leaderboard
+        run: python validate_submission.py rebuild-leaderboard
+
+      - name: Commit leaderboard update
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add experiments/leaderboard.json experiments/README.md
+          git diff --cached --quiet || git commit -m "Update leaderboard"
+          git push

flyai_bench-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+*.local.yaml
+dist/
+__pycache__/
+.venv/
+e2e_test_results/
+*.egg-info/

flyai_bench-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 FlyaiLab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

flyai_bench-0.1.0/PKG-INFO

@@ -0,0 +1,200 @@
+Metadata-Version: 2.5
+Name: flyai-bench
+Version: 0.1.0
+Summary: E-Commerce Last Exam: Agent benchmark for travel and e-commerce tool-use tasks
+Author-email: FlyaiLab <flyaimainsite@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: huggingface-hub>=0.20
+Requires-Dist: openai>=1.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# flyai-bench
+
+![Python](https://img.shields.io/badge/python-%3E%3D3.10-blue)
+![License](https://img.shields.io/badge/license-MIT-green)
+
+An evaluation framework for LLM agents. It loads evaluation tasks from a
+HuggingFace dataset, runs an agent plus verifier inside isolated Docker
+containers, and produces standardized scores (reward in the range `0.00 – 1.00`).
+
+## Prerequisites
+
+- Python >= 3.10
+- Docker (with access to the image registry that hosts the benchmark images;
+  the target platform is `linux/amd64`)
+- An OpenAI-compatible LLM endpoint for the agent, and one for the judge/verifier
+  (they can be the same endpoint)
+
+## Installation
+
+```bash
+# From a clone of this repository
+pip install -e .
+
+# Or install the runtime dependencies directly
+pip install pyyaml huggingface_hub openai
+```
+
+Installing the package exposes a `flyai-bench` console command. The examples
+below also work by invoking `run_eval.py` directly.
+
+## Quick start
+
+```bash
+# 1. Copy the default config and fill in your values
+cp eval_config.yaml eval_config.local.yaml
+#    edit eval_config.local.yaml: set the image registry, the LLM/judge
+#    base URLs, API keys, and the model name
+
+# 2. Run the evaluation
+python run_eval.py run --config eval_config.local.yaml
+
+# 3. Inspect results
+python run_eval.py report --config eval_config.local.yaml
+```
+
+## Supported benchmarks
+
+| Benchmark | Dataset | Tasks | Description |
+|-----------|---------|-------|-------------|
+| ecommerce_last_exam | [FlyaiLab/ecommerce_last_exam](https://huggingface.co/datasets/FlyaiLab/ecommerce_last_exam) | 1000 | Travel + e-commerce agent tool-use evaluation (500 travel, 500 e-commerce) |
+
+## Configuration
+
+Key fields in `eval_config.yaml` (copy to `eval_config.local.yaml` before editing):
+
+| Field | Meaning |
+|-------|---------|
+| `dataset.repo_id` / `dataset.config` / `dataset.split` | HuggingFace dataset, config (`travel` / `e_commerce`), and split |
+| `docker.registry` | Image registry that hosts the benchmark images. Set to your own registry, or override at runtime with `--registry`. Leave empty to use image names as-is. |
+| `docker.platform` | Container platform (default `linux/amd64`) |
+| `agent.cmd` | Command that runs the agent inside the container |
+| `agent.llm_base_url` / `agent.llm_api_key` / `agent.llm_model` | OpenAI-compatible endpoint, key, and model for the agent |
+| `verifier.judge_base_url` / `judge_api_key` / `judge_model` | Endpoint, key, and model for the judge/verifier |
+| `runner.concurrency` / `runner.limit` / `runner.skip_done` | Parallelism, cap on number of instances (`null` = all), and whether to skip instances that already have a reward |
+
+> API keys are read from your local config and injected into the containers as
+> environment variables. Never commit `eval_config.local.yaml` — it is already
+> in `.gitignore`.
+
+## Evaluation flow
+
+```
+HuggingFace Dataset
+    │
+    ▼
+┌─────────────────────────────────┐
+│  For each instance:             │
+│  1. docker pull <image>         │
+│  2. docker run (start env)      │
+│  3. agent calls tools -> answer │
+│  4. verifier (test.sh) scores   │
+│  5. emit reward.txt (0 – 1)     │
+└─────────────────────────────────┘
+    │
+    ▼
+scores.jsonl + summary.json
+```
+
+## CLI commands
+
+```bash
+# Run the evaluation
+python run_eval.py run [--dataset-config travel|e_commerce] [--limit N] [--dry-run]
+
+# Check progress
+python run_eval.py status
+
+# Generate a report
+python run_eval.py report
+
+# Package results for leaderboard submission
+python run_eval.py submit --model deepseek-v4-flash --provider deepseek
+```
+
+## Output
+
+| File | Contents |
+|------|----------|
+| `scores.jsonl` | One line per instance: `{instance_id, domain, reward, duration_sec}` |
+| `summary.json` | Aggregate statistics (`avg_reward`, breakdown `by_domain`) |
+
+`reward` is a float in `0.00 – 1.00` produced by the verifier for each instance.
+
+## Submitting results to the leaderboard
+
+1. After evaluating, package the results with `submit`:
+
+   ```bash
+   python run_eval.py submit \
+     --dataset-config travel \
+     --model deepseek-v4-flash \
+     --provider deepseek \
+     --agent-type mini-swe-agent
+   ```
+
+2. Review `experiments/evaluation/travel/<slug>/metadata.yaml` and complete the
+   model and agent details.
+
+3. Open a pull request to the `main` branch of the flyai-bench repository.
+
+4. CI validates the format automatically; once merged, the leaderboard updates.
+
+### Submission package layout
+
+```
+evaluation/travel/20260830_deepseek-v4-flash_mini-swe-agent/
+├── metadata.yaml      # model/agent info + evaluation statistics
+├── scores.jsonl       # per instance: {instance_id, domain, reward, duration_sec}
+└── summary.json       # aggregate statistics (avg_reward, by_domain)
+```
+
+## Writing a custom agent
+
+Implement a script that, inside the container:
+
+1. Reads `/app/tool_defs.json` for the tool definitions
+2. Reads `/app/system.md` + `/app/instruction.md` for the task
+3. Calls `/app/tools/<name> --arg val` to execute a tool
+4. Writes the result to `/app/answer.json`
+
+Point the runner at your agent with `--agent-cmd`:
+
+```bash
+python run_eval.py run --agent-cmd "python /app/my_agent.py"
+```
+
+`agent.py` (a minimal LLM agent) and `mini_swe_agent.py` (a terminal/bash agent)
+are provided as reference implementations.
+
+## Project structure
+
+```
+flyai-bench/
+├── README.md
+├── pyproject.toml            # packaging (installs the `flyai-bench` CLI)
+├── run_eval.py               # evaluation CLI (run / status / report / submit)
+├── eval_config.yaml          # default config (copy to eval_config.local.yaml)
+├── benchmark.yaml            # benchmark metadata
+├── agent.py                  # reference LLM agent
+├── mini_swe_agent.py         # reference terminal agent
+├── tool_server.py            # in-sandbox tool proxy (permission isolation)
+├── sandbox_setup.sh          # container permission setup
+├── validate_submission.py    # submission validation + leaderboard rebuild
+├── experiments/              # submitted results + leaderboard.json
+├── leaderboard_space/        # HuggingFace Space (leaderboard UI)
+└── src/flyai_bench/          # installable package mirror of the above
+```
+
+> Note: the top-level scripts and the `src/flyai_bench/` package currently hold
+> parallel copies of the same code. Prefer editing one and keeping them in sync
+> (or consolidating on the package) to avoid drift.
+
+## License
+
+Released under the MIT license. See [LICENSE](LICENSE) for details.

flyai_bench-0.1.0/README.md

@@ -0,0 +1,185 @@
+# flyai-bench
+
+![Python](https://img.shields.io/badge/python-%3E%3D3.10-blue)
+![License](https://img.shields.io/badge/license-MIT-green)
+
+An evaluation framework for LLM agents. It loads evaluation tasks from a
+HuggingFace dataset, runs an agent plus verifier inside isolated Docker
+containers, and produces standardized scores (reward in the range `0.00 – 1.00`).
+
+## Prerequisites
+
+- Python >= 3.10
+- Docker (with access to the image registry that hosts the benchmark images;
+  the target platform is `linux/amd64`)
+- An OpenAI-compatible LLM endpoint for the agent, and one for the judge/verifier
+  (they can be the same endpoint)
+
+## Installation
+
+```bash
+# From a clone of this repository
+pip install -e .
+
+# Or install the runtime dependencies directly
+pip install pyyaml huggingface_hub openai
+```
+
+Installing the package exposes a `flyai-bench` console command. The examples
+below also work by invoking `run_eval.py` directly.
+
+## Quick start
+
+```bash
+# 1. Copy the default config and fill in your values
+cp eval_config.yaml eval_config.local.yaml
+#    edit eval_config.local.yaml: set the image registry, the LLM/judge
+#    base URLs, API keys, and the model name
+
+# 2. Run the evaluation
+python run_eval.py run --config eval_config.local.yaml
+
+# 3. Inspect results
+python run_eval.py report --config eval_config.local.yaml
+```
+
+## Supported benchmarks
+
+| Benchmark | Dataset | Tasks | Description |
+|-----------|---------|-------|-------------|
+| ecommerce_last_exam | [FlyaiLab/ecommerce_last_exam](https://huggingface.co/datasets/FlyaiLab/ecommerce_last_exam) | 1000 | Travel + e-commerce agent tool-use evaluation (500 travel, 500 e-commerce) |
+
+## Configuration
+
+Key fields in `eval_config.yaml` (copy to `eval_config.local.yaml` before editing):
+
+| Field | Meaning |
+|-------|---------|
+| `dataset.repo_id` / `dataset.config` / `dataset.split` | HuggingFace dataset, config (`travel` / `e_commerce`), and split |
+| `docker.registry` | Image registry that hosts the benchmark images. Set to your own registry, or override at runtime with `--registry`. Leave empty to use image names as-is. |
+| `docker.platform` | Container platform (default `linux/amd64`) |
+| `agent.cmd` | Command that runs the agent inside the container |
+| `agent.llm_base_url` / `agent.llm_api_key` / `agent.llm_model` | OpenAI-compatible endpoint, key, and model for the agent |
+| `verifier.judge_base_url` / `judge_api_key` / `judge_model` | Endpoint, key, and model for the judge/verifier |
+| `runner.concurrency` / `runner.limit` / `runner.skip_done` | Parallelism, cap on number of instances (`null` = all), and whether to skip instances that already have a reward |
+
+> API keys are read from your local config and injected into the containers as
+> environment variables. Never commit `eval_config.local.yaml` — it is already
+> in `.gitignore`.
+
+## Evaluation flow
+
+```
+HuggingFace Dataset
+    │
+    ▼
+┌─────────────────────────────────┐
+│  For each instance:             │
+│  1. docker pull <image>         │
+│  2. docker run (start env)      │
+│  3. agent calls tools -> answer │
+│  4. verifier (test.sh) scores   │
+│  5. emit reward.txt (0 – 1)     │
+└─────────────────────────────────┘
+    │
+    ▼
+scores.jsonl + summary.json
+```
+
+## CLI commands
+
+```bash
+# Run the evaluation
+python run_eval.py run [--dataset-config travel|e_commerce] [--limit N] [--dry-run]
+
+# Check progress
+python run_eval.py status
+
+# Generate a report
+python run_eval.py report
+
+# Package results for leaderboard submission
+python run_eval.py submit --model deepseek-v4-flash --provider deepseek
+```
+
+## Output
+
+| File | Contents |
+|------|----------|
+| `scores.jsonl` | One line per instance: `{instance_id, domain, reward, duration_sec}` |
+| `summary.json` | Aggregate statistics (`avg_reward`, breakdown `by_domain`) |
+
+`reward` is a float in `0.00 – 1.00` produced by the verifier for each instance.
+
+## Submitting results to the leaderboard
+
+1. After evaluating, package the results with `submit`:
+
+   ```bash
+   python run_eval.py submit \
+     --dataset-config travel \
+     --model deepseek-v4-flash \
+     --provider deepseek \
+     --agent-type mini-swe-agent
+   ```
+
+2. Review `experiments/evaluation/travel/<slug>/metadata.yaml` and complete the
+   model and agent details.
+
+3. Open a pull request to the `main` branch of the flyai-bench repository.
+
+4. CI validates the format automatically; once merged, the leaderboard updates.
+
+### Submission package layout
+
+```
+evaluation/travel/20260830_deepseek-v4-flash_mini-swe-agent/
+├── metadata.yaml      # model/agent info + evaluation statistics
+├── scores.jsonl       # per instance: {instance_id, domain, reward, duration_sec}
+└── summary.json       # aggregate statistics (avg_reward, by_domain)
+```
+
+## Writing a custom agent
+
+Implement a script that, inside the container:
+
+1. Reads `/app/tool_defs.json` for the tool definitions
+2. Reads `/app/system.md` + `/app/instruction.md` for the task
+3. Calls `/app/tools/<name> --arg val` to execute a tool
+4. Writes the result to `/app/answer.json`
+
+Point the runner at your agent with `--agent-cmd`:
+
+```bash
+python run_eval.py run --agent-cmd "python /app/my_agent.py"
+```
+
+`agent.py` (a minimal LLM agent) and `mini_swe_agent.py` (a terminal/bash agent)
+are provided as reference implementations.
+
+## Project structure
+
+```
+flyai-bench/
+├── README.md
+├── pyproject.toml            # packaging (installs the `flyai-bench` CLI)
+├── run_eval.py               # evaluation CLI (run / status / report / submit)
+├── eval_config.yaml          # default config (copy to eval_config.local.yaml)
+├── benchmark.yaml            # benchmark metadata
+├── agent.py                  # reference LLM agent
+├── mini_swe_agent.py         # reference terminal agent
+├── tool_server.py            # in-sandbox tool proxy (permission isolation)
+├── sandbox_setup.sh          # container permission setup
+├── validate_submission.py    # submission validation + leaderboard rebuild
+├── experiments/              # submitted results + leaderboard.json
+├── leaderboard_space/        # HuggingFace Space (leaderboard UI)
+└── src/flyai_bench/          # installable package mirror of the above
+```
+
+> Note: the top-level scripts and the `src/flyai_bench/` package currently hold
+> parallel copies of the same code. Prefer editing one and keeping them in sync
+> (or consolidating on the package) to avoid drift.
+
+## License
+
+Released under the MIT license. See [LICENSE](LICENSE) for details.

flyai_bench-0.1.0/agent.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Minimal reference agent for E-Commerce Last Exam benchmark.
+
+This file demonstrates the agent/environment interface. In real evaluations it can be replaced by:
+- mini-swe-agent (general-purpose CLI agent)
+- a custom LLM agent (calling a model via an OpenAI-compatible API)
+
+Agent responsibilities:
+1. Read /app/tool_defs.json for the available tool definitions (OpenAI function-calling schema)
+2. Read system.md + instruction.md for the task prompt
+3. Call the LLM to decide -> call tools -> loop until the task is done
+4. Write the final result to /app/answer.json
+
+Tool call convention:
+  /app/tools/<tool_name> --param1 value1 --param2 value2
+  Output is JSON (stdout)
+"""
+import json
+import os
+import subprocess
+import sys
+
+# Environment file paths (inside the container)
+TOOL_DEFS_PATH = "/app/tool_defs.json"
+SYSTEM_MD_PATH = "/app/system.md"
+INSTRUCTION_MD_PATH = "/app/instruction.md"
+ANSWER_PATH = "/app/answer.json"
+TOOL_CALL_LOG = os.environ.get("TOOL_CALL_LOG", "/app/.tool_calls.jsonl")
+
+# Agent LLM config (injected via environment variables)
+LLM_BASE_URL = os.environ.get("LLM_BASE_URL", "http://localhost:4000")
+LLM_API_KEY = os.environ.get("LLM_API_KEY", "")
+LLM_MODEL = os.environ.get("LLM_MODEL", "qwen3.7-plus")
+
+
+def load_tool_defs():
+    with open(TOOL_DEFS_PATH, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def load_prompts():
+    system = ""
+    instruction = ""
+    if os.path.exists(SYSTEM_MD_PATH):
+        with open(SYSTEM_MD_PATH, encoding="utf-8") as f:
+            system = f.read()
+    if os.path.exists(INSTRUCTION_MD_PATH):
+        with open(INSTRUCTION_MD_PATH, encoding="utf-8") as f:
+            instruction = f.read()
+    return system, instruction
+
+
+def call_tool(tool_name, arguments):
+    """Execute a single tool call and return the output JSON."""
+    tool_path = f"/app/tools/{tool_name}"
+    if not os.path.exists(tool_path):
+        return {"error": f"tool not found: {tool_name}"}
+
+    cmd = [tool_path]
+    for k, v in arguments.items():
+        cmd += [f"--{k}", str(v)]
+
+    try:
+        r = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
+        if r.returncode != 0:
+            return {"error": r.stderr[:500]}
+        try:
+            return json.loads(r.stdout)
+        except json.JSONDecodeError:
+            return {"raw_output": r.stdout[:2000]}
+    except subprocess.TimeoutExpired:
+        return {"error": "tool call timeout"}
+
+
+def log_tool_call(tool_name, arguments, result):
+    """Log a tool call."""
+    entry = {"tool": tool_name, "arguments": arguments, "result": result}
+    with open(TOOL_CALL_LOG, "a", encoding="utf-8") as f:
+        f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+
+def call_llm(messages, tools):
+    """Call the LLM API (OpenAI-compatible)."""
+    try:
+        import openai
+    except ImportError:
+        print("ERROR: openai package not installed", file=sys.stderr)
+        sys.exit(1)
+
+    client = openai.OpenAI(base_url=LLM_BASE_URL, api_key=LLM_API_KEY)
+    response = client.chat.completions.create(
+        model=LLM_MODEL,
+        messages=messages,
+        tools=tools,
+        tool_choice="auto",
+        temperature=0.1,
+    )
+    return response.choices[0].message
+
+
+def run():
+    """Main agent loop."""
+    tool_defs = load_tool_defs()
+    system_prompt, instruction = load_prompts()
+
+    # Build the OpenAI tools format
+    tools = []
+    for td in tool_defs:
+        tools.append({
+            "type": "function",
+            "function": {
+                "name": td["name"],
+                "description": td.get("description", ""),
+                "parameters": td.get("parameters", {}),
+            }
+        })
+
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": instruction},
+    ]
+
+    max_turns = 20
+    for turn in range(max_turns):
+        response = call_llm(messages, tools)
+
+        if response.tool_calls:
+            messages.append(response.model_dump())
+            for tc in response.tool_calls:
+                fn_name = tc.function.name
+                try:
+                    fn_args = json.loads(tc.function.arguments)
+                except json.JSONDecodeError:
+                    fn_args = {}
+
+                result = call_tool(fn_name, fn_args)
+                log_tool_call(fn_name, fn_args, result)
+
+                messages.append({
+                    "role": "tool",
+                    "tool_call_id": tc.id,
+                    "content": json.dumps(result, ensure_ascii=False),
+                })
+        else:
+            # Agent done; parse the final reply into answer.json
+            content = response.content or ""
+            try:
+                answer = json.loads(content)
+            except json.JSONDecodeError:
+                answer = {"raw_response": content}
+
+            with open(ANSWER_PATH, "w", encoding="utf-8") as f:
+                json.dump(answer, f, ensure_ascii=False, indent=2)
+            print(f"Agent completed in {turn + 1} turns. Answer written to {ANSWER_PATH}")
+            return
+
+    print(f"WARNING: Agent reached max turns ({max_turns}) without final answer",
+          file=sys.stderr)
+
+
+if __name__ == "__main__":
+    run()

flyai_bench-0.1.0/benchmark.yaml

@@ -0,0 +1,49 @@
+name: ecommerce_last_exam
+version: "1.0"
+description: "E-Commerce Last Exam: Agent benchmark for travel and e-commerce tool-use tasks"
+
+dataset:
+  repo: FlyaiLab/ecommerce_last_exam
+  configs:
+    - travel      # 500 Chinese trip_guide tasks (hotel/traffic/poi)
+    - e_commerce  # 500 English shopping tasks (fashion/health/electronics/outdoor/home)
+  split: test
+
+agent:
+  type: mini-swe-agent
+  timeout_sec: 1800
+  interface: tool-calling
+  output_file: /app/answer.json
+
+verifier:
+  timeout_sec: 1200
+  test_script: /app/tests/test.sh
+  reward_file: /logs/verifier/reward.txt
+  env:
+    JUDGE_BASE_URL: "${JUDGE_BASE_URL}"
+    JUDGE_API_KEY: "${JUDGE_API_KEY}"
+    JUDGE_MODEL: "${JUDGE_MODEL}"
+
+docker:
+  registry: ""
+  platform: linux/amd64
+
+output:
+  scores_file: scores.jsonl
+  summary_file: summary.json
+  reward_format: float  # 0.00 - 1.00
+
+submission:
+  experiments_dir: experiments/evaluation/
+  format:
+    required:
+      - metadata.yaml    # model/agent info + evaluation stats
+      - scores.jsonl     # per-instance: {instance_id, domain, reward, duration_sec}
+    optional:
+      - summary.json     # aggregate stats (auto-generated if missing)
+      - trajs/           # per-instance agent traces
+  workflow:
+    - "python run_eval.py run --dataset-config travel"
+    - "python run_eval.py report --dataset-config travel"
+    - "python run_eval.py submit --dataset-config travel --model your-model"
+    - "# then PR to experiments repo"