llmdep 0.2.0__tar.gz

llmdep-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Anonymous
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llmdep-0.2.0/MANIFEST.in

@@ -0,0 +1,33 @@
+include README.md
+include LICENSE
+include setup.py
+include setup.cfg
+include MANIFEST.in
+
+recursive-include mep_client *.py *.json
+recursive-exclude mep_client/final_results *
+recursive-exclude mep_client/log *
+
+include llm/BaseModel.py
+include llm/__init__.py
+include method_server/*.py
+recursive-include llmdep_frontend *.py *.html *.js *.css
+
+recursive-include method_server/eval *.py *.txt *.md
+recursive-include dep_server *.py *.md
+
+recursive-exclude method_server *.csv *.jsonl *.parquet *.arrow *.pkl *.pt *.pth *.safetensors
+recursive-exclude method_server/*/cache *
+recursive-exclude method_server/*/data *
+
+global-exclude config_backup.py
+global-exclude __pycache__
+global-exclude *.pyc
+global-exclude *.pyo
+global-exclude *.pyd
+global-exclude *.parquet
+global-exclude *.arrow
+global-exclude .gitignore
+global-exclude .git
+global-exclude .idea
+global-exclude *.egg-info

llmdep-0.2.0/PKG-INFO

@@ -0,0 +1,233 @@
+Metadata-Version: 2.1
+Name: llmdep
+Version: 0.2.0
+Summary: Distributed LLM Evaluation Benchmark Server
+Home-page: https://github.com/tjunlp-lab/DEP
+Author: TJUNLP
+License: MIT
+Platform: UNKNOWN
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# llmdep - Distributed LLM Evaluation Benchmark Server
+
+[![Python](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/)
+[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+A distributed benchmark server for evaluating Large Language Models (LLMs) across multiple benchmarks.
+
+## Features
+
+- **Distributed Evaluation**: Support concurrent worker processes
+- **Multiple Benchmarks**: ARC, ETHICS, MMLU-Pro, BiPaR, and more
+- **Rate Limiting**: Token bucket based rate limiting for API calls
+- **Batch Processing**: Process multiple model-benchmark pairs in batch
+- **Resume Support**: Continue interrupted tasks with `--run_id`
+
+## Installation
+
+```bash
+pip install llmdep
+```
+
+Or install from source:
+
+```bash
+pip install -e .
+```
+
+## Quick Start
+
+### Configure Benchmark Workspace
+
+Benchmarks should live outside the installed pip package. Configure a benchmark root once:
+
+```bash
+llmdep config set dataset_root /data/eval_datasets
+```
+
+The directory should contain benchmark folders directly:
+
+```text
+/data/eval_datasets/
+├── BiPaR/
+│   ├── dataset_card.json
+│   ├── methods/
+│   ├── data/
+│   └── cache/
+└── gsm8k/
+    ├── dataset_card.json
+    ├── methods/
+    ├── data/
+    └── cache/
+```
+
+Check the active benchmark root:
+
+```bash
+llmdep list benchmark
+```
+
+Configure a model root once as well:
+
+```bash
+llmdep config set model_root /data/llm_models
+```
+
+The directory should contain only user model folders directly. `BaseModel` is not placed here; it remains inside the installed `llmdep` package.
+
+```text
+/data/llm_models/
+└── qwen-plus/
+    ├── model.py
+    └── model_card.json
+```
+
+Each `model.py` should expose a `Model` class with `call_with_prompt(...)`. Import the base class from the installed package:
+
+```python
+from llm.BaseModel import BaseModel
+
+class Model(BaseModel):
+    def call_with_prompt(self, input_data):
+        ...
+```
+
+### Single Task
+
+```bash
+# Generate answers
+llmdep gen_eval -b ARC -m qwen-plus -w 10
+
+# Generate answers and evaluate
+llmdep gen_eval -b ARC -m qwen-plus -w 10 --evaluate
+```
+
+### Batch Processing
+
+Create a task configuration file `tasks.json`:
+
+```json
+[
+    {"model": "qwen-plus", "benchmark": "ARC_parquet"},
+    {"model": "deepseek", "benchmark": "ethics"}
+]
+```
+
+Then run:
+
+```bash
+llmdep batch --config tasks.json -w 10
+```
+
+### Resume Interrupted Task
+
+```bash
+llmdep gen_eval --run_id 20260210-164048-322281
+```
+
+## Commands
+
+### gen_eval
+
+Generate model answers and optionally evaluate.
+
+```bash
+llmdep gen_eval [options]
+
+Options:
+  -b, --benchmark     Benchmark name (e.g., ARC, ethics)
+  -m, --model         Model name (e.g., qwen-plus, deepseek)
+  -w, --worker_nums   Number of concurrent workers (default: 2)
+  --batch_size        Batch size (default: 1)
+  --rate_limit        Token bucket fill rate (default: 5.0)
+  --bucket_capacity   Token bucket capacity (default: 10.0)
+  --limited_test      Quick test with first 10 questions, or pass N
+  --evaluate         Run evaluation after generation
+  --run_id          Resume from run_id
+```
+
+### evaluate
+
+Run evaluation on existing results.
+
+```bash
+llmdep evaluate -b ARC -r 20260210-164048-322281_qwen-plus
+```
+
+### list
+
+List available benchmarks or models.
+
+```bash
+llmdep list benchmark
+llmdep list model
+```
+
+### dashboard
+
+Start the local web control panel.
+
+```bash
+llmdep dashboard
+llmdep dashboard --host 127.0.0.1 --port 8765
+```
+
+### batch
+
+Batch process tasks from config file.
+
+```bash
+llmdep batch --config tasks.json
+```
+
+## Project Structure
+
+```
+llmdep/
+├── llm/                  # Packaged BaseModel compatibility module
+├── mep_client/           # Client for generating answers
+│   ├── run/
+│   └── API/
+├── method_server/      # Benchmark implementations
+│   ├── ARC_parquet/
+│   ├── ethics/
+│   ├── BiPaR/
+│   └── ...
+└── llmdep_cli.py       # CLI entry point
+```
+
+## Adding New Models
+
+1. Configure `model_root`: `llmdep config set model_root /data/llm_models`
+2. Create folder under `/data/llm_models/<your_model>/`
+2. Implement `model.py` inheriting from `BaseModel`
+3. Create `model_card.json`
+
+## Adding New Benchmarks
+
+1. Configure `dataset_root`: `llmdep config set dataset_root /data/eval_datasets`
+1. Create folder under `/data/eval_datasets/<benchmark_name>/`
+2. Implement `prepare_ques.py` - Prepare questions and golden answers
+3. Implement `eval_script.py` - Run evaluation
+4. Create `dataset_card.json`
+
+## License
+
+MIT License
+
+

llmdep-0.2.0/README.md

@@ -0,0 +1,206 @@
+# llmdep - Distributed LLM Evaluation Benchmark Server
+
+[![Python](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/)
+[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+A distributed benchmark server for evaluating Large Language Models (LLMs) across multiple benchmarks.
+
+## Features
+
+- **Distributed Evaluation**: Support concurrent worker processes
+- **Multiple Benchmarks**: ARC, ETHICS, MMLU-Pro, BiPaR, and more
+- **Rate Limiting**: Token bucket based rate limiting for API calls
+- **Batch Processing**: Process multiple model-benchmark pairs in batch
+- **Resume Support**: Continue interrupted tasks with `--run_id`
+
+## Installation
+
+```bash
+pip install llmdep
+```
+
+Or install from source:
+
+```bash
+pip install -e .
+```
+
+## Quick Start
+
+### Configure Benchmark Workspace
+
+Benchmarks should live outside the installed pip package. Configure a benchmark root once:
+
+```bash
+llmdep config set dataset_root /data/eval_datasets
+```
+
+The directory should contain benchmark folders directly:
+
+```text
+/data/eval_datasets/
+├── BiPaR/
+│   ├── dataset_card.json
+│   ├── methods/
+│   ├── data/
+│   └── cache/
+└── gsm8k/
+    ├── dataset_card.json
+    ├── methods/
+    ├── data/
+    └── cache/
+```
+
+Check the active benchmark root:
+
+```bash
+llmdep list benchmark
+```
+
+Configure a model root once as well:
+
+```bash
+llmdep config set model_root /data/llm_models
+```
+
+The directory should contain only user model folders directly. `BaseModel` is not placed here; it remains inside the installed `llmdep` package.
+
+```text
+/data/llm_models/
+└── qwen-plus/
+    ├── model.py
+    └── model_card.json
+```
+
+Each `model.py` should expose a `Model` class with `call_with_prompt(...)`. Import the base class from the installed package:
+
+```python
+from llm.BaseModel import BaseModel
+
+class Model(BaseModel):
+    def call_with_prompt(self, input_data):
+        ...
+```
+
+### Single Task
+
+```bash
+# Generate answers
+llmdep gen_eval -b ARC -m qwen-plus -w 10
+
+# Generate answers and evaluate
+llmdep gen_eval -b ARC -m qwen-plus -w 10 --evaluate
+```
+
+### Batch Processing
+
+Create a task configuration file `tasks.json`:
+
+```json
+[
+    {"model": "qwen-plus", "benchmark": "ARC_parquet"},
+    {"model": "deepseek", "benchmark": "ethics"}
+]
+```
+
+Then run:
+
+```bash
+llmdep batch --config tasks.json -w 10
+```
+
+### Resume Interrupted Task
+
+```bash
+llmdep gen_eval --run_id 20260210-164048-322281
+```
+
+## Commands
+
+### gen_eval
+
+Generate model answers and optionally evaluate.
+
+```bash
+llmdep gen_eval [options]
+
+Options:
+  -b, --benchmark     Benchmark name (e.g., ARC, ethics)
+  -m, --model         Model name (e.g., qwen-plus, deepseek)
+  -w, --worker_nums   Number of concurrent workers (default: 2)
+  --batch_size        Batch size (default: 1)
+  --rate_limit        Token bucket fill rate (default: 5.0)
+  --bucket_capacity   Token bucket capacity (default: 10.0)
+  --limited_test      Quick test with first 10 questions, or pass N
+  --evaluate         Run evaluation after generation
+  --run_id          Resume from run_id
+```
+
+### evaluate
+
+Run evaluation on existing results.
+
+```bash
+llmdep evaluate -b ARC -r 20260210-164048-322281_qwen-plus
+```
+
+### list
+
+List available benchmarks or models.
+
+```bash
+llmdep list benchmark
+llmdep list model
+```
+
+### dashboard
+
+Start the local web control panel.
+
+```bash
+llmdep dashboard
+llmdep dashboard --host 127.0.0.1 --port 8765
+```
+
+### batch
+
+Batch process tasks from config file.
+
+```bash
+llmdep batch --config tasks.json
+```
+
+## Project Structure
+
+```
+llmdep/
+├── llm/                  # Packaged BaseModel compatibility module
+├── mep_client/           # Client for generating answers
+│   ├── run/
+│   └── API/
+├── method_server/      # Benchmark implementations
+│   ├── ARC_parquet/
+│   ├── ethics/
+│   ├── BiPaR/
+│   └── ...
+└── llmdep_cli.py       # CLI entry point
+```
+
+## Adding New Models
+
+1. Configure `model_root`: `llmdep config set model_root /data/llm_models`
+2. Create folder under `/data/llm_models/<your_model>/`
+2. Implement `model.py` inheriting from `BaseModel`
+3. Create `model_card.json`
+
+## Adding New Benchmarks
+
+1. Configure `dataset_root`: `llmdep config set dataset_root /data/eval_datasets`
+1. Create folder under `/data/eval_datasets/<benchmark_name>/`
+2. Implement `prepare_ques.py` - Prepare questions and golden answers
+3. Implement `eval_script.py` - Run evaluation
+4. Create `dataset_card.json`
+
+## License
+
+MIT License

llmdep-0.2.0/llm/BaseModel.py

@@ -0,0 +1,40 @@
+import os
+import logging
+import datetime
+
+class BaseModel:
+    def __init__(self, args):
+        self.args = args
+        self.worker_num = getattr(args, 'worker_nums', 1)
+        self.model_name = getattr(args, 'model_name', getattr(args, 'modelname', None))
+        run_id = getattr(args, 'run_id', None)
+        name = self.model_name if self.model_name else 'model'
+        if run_id:
+            self.run_tag = f"{run_id}_{name}"
+        else:
+            ts = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+            self.run_tag = f"{ts}_{name}"
+        self._set_logger()
+
+    def _set_logger(self):
+        self.generation_logging = logging.getLogger(f"generation_logging_{self.run_tag}")
+        self.error_logging = logging.getLogger(f"error_logging_{self.run_tag}")
+        project_path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+        log_dir = os.path.join(project_path, 'mep_client', 'log', self.run_tag)
+        os.makedirs(log_dir, exist_ok=True)
+        for logger in (self.generation_logging, self.error_logging):
+            for handler in logger.handlers[:]:
+                handler.close()
+                logger.removeHandler(handler)
+        generation_handler = logging.FileHandler(os.path.join(log_dir, 'generation.log'), mode='a', encoding='utf-8')
+        error_handler = logging.FileHandler(os.path.join(log_dir, 'error.log'), mode='a', encoding='utf-8')
+        formatter = logging.Formatter('\n%(asctime)s %(levelname)s\n%(message)s')
+        generation_handler.setFormatter(formatter)
+        error_handler.setFormatter(formatter)
+        self.generation_logging.setLevel(logging.INFO)
+        self.error_logging.setLevel(logging.INFO)
+        self.generation_logging.addHandler(generation_handler)
+        self.error_logging.addHandler(error_handler)
+
+    def list_prompts_to_questions(self, prompts):
+        return [[{"role": "user", "content": p}] for p in prompts]

llmdep-0.2.0/llm/__init__.py

@@ -0,0 +1 @@
+from .BaseModel import BaseModel