llm_batch_helper 0.1.4__tar.gz → 0.1.6__tar.gz

{llm_batch_helper-0.1.4 → llm_batch_helper-0.1.6}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: llm_batch_helper
-Version: 0.1.4
+Version: 0.1.6
 Summary: A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities and response caching.
 License: MIT
-Keywords: llm,openai,batch,async,ai,nlp,api
+Keywords: llm,openai,together,batch,async,ai,nlp,api
 Author: Tianyi Peng
 Author-email: tianyipeng95@gmail.com
 Requires-Python: >=3.11,<4.0
@@ -19,7 +19,6 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: httpx (>=0.24.0,<2.0.0)
 Requires-Dist: openai (>=1.0.0,<2.0.0)
-Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: tenacity (>=8.0.0,<9.0.0)
 Requires-Dist: tqdm (>=4.65.0,<5.0.0)
 Project-URL: Homepage, https://github.com/TianyiPeng/LLM_batch_helper
@@ -28,14 +27,36 @@ Description-Content-Type: text/markdown
 
 # LLM Batch Helper
 
-A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities and response caching.
+[![PyPI version](https://badge.fury.io/py/llm_batch_helper.svg)](https://badge.fury.io/py/llm_batch_helper)
+[![Downloads](https://pepy.tech/badge/llm_batch_helper)](https://pepy.tech/project/llm_batch_helper)
+[![Downloads/Month](https://pepy.tech/badge/llm_batch_helper/month)](https://pepy.tech/project/llm_batch_helper)
+[![Documentation Status](https://readthedocs.org/projects/llm-batch-helper/badge/?version=latest)](https://llm-batch-helper.readthedocs.io/en/latest/?badge=latest)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities, response caching, prompt verification, and more. This package is designed to streamline applications like LLM simulation, LLM-as-a-judge, and other batch processing scenarios.
+
+📖 **[Complete Documentation](https://llm-batch-helper.readthedocs.io/)** | 🚀 **[Quick Start Guide](https://llm-batch-helper.readthedocs.io/en/latest/quickstart.html)**
+
+## Why we designed this package
+
+Calling LLM APIs has become increasingly common, but several pain points exist in practice:
+
+1. **Efficient Batch Processing**: How do you run LLM calls in batches efficiently? Our async implementation is 3X-100X faster than multi-thread/multi-process approaches.
+
+2. **API Reliability**: LLM APIs can be unstable, so we need robust retry mechanisms when calls get interrupted.
+
+3. **Long-Running Simulations**: During long-running LLM simulations, computers can crash and APIs can fail. Can we cache LLM API calls to avoid repeating completed work?
+
+4. **Output Validation**: LLM outputs often have format requirements. If the output isn't right, we need to retry with validation.
+
+This package is designed to solve these exact pain points with async processing, intelligent caching, and comprehensive error handling. If there are some additional features you need, please post an issue.  
 
 ## Features
 
 - **Async Processing**: Submit multiple prompts concurrently for faster processing
 - **Response Caching**: Automatically cache responses to avoid redundant API calls
 - **Multiple Input Formats**: Support for both file-based and list-based prompts
-- **Provider Support**: Works with OpenAI API
+- **Provider Support**: Works with OpenAI and Together.ai APIs
 - **Retry Logic**: Built-in retry mechanism with exponential backoff
 - **Verification Callbacks**: Custom verification for response quality
 - **Progress Tracking**: Real-time progress bars for batch operations
@@ -67,9 +88,29 @@ poetry shell
 
 ### 1. Set up environment variables
 
+**Option A: Environment Variables**
 ```bash
 # For OpenAI
 export OPENAI_API_KEY="your-openai-api-key"
+
+# For Together.ai
+export TOGETHER_API_KEY="your-together-api-key"
+```
+
+**Option B: .env File (Recommended for Development)**
+```python
+# In your script, before importing llm_batch_helper
+from dotenv import load_dotenv
+load_dotenv()  # Load from .env file
+
+# Then use the package normally
+from llm_batch_helper import LLMConfig, process_prompts_batch
+```
+
+Create a `.env` file in your project:
+```
+OPENAI_API_KEY=your-openai-api-key
+TOGETHER_API_KEY=your-together-api-key
 ```
 
 ### 2. Interactive Tutorial (Recommended)
@@ -82,14 +123,18 @@ The tutorial covers all features with interactive examples!
 
 ```python
 import asyncio
+from dotenv import load_dotenv  # Optional: for .env file support
 from llm_batch_helper import LLMConfig, process_prompts_batch
 
+# Optional: Load environment variables from .env file
+load_dotenv()
+
 async def main():
     # Create configuration
     config = LLMConfig(
         model_name="gpt-4o-mini",
         temperature=0.7,
-        max_tokens=100,
+        max_completion_tokens=100,  # or use max_tokens for backward compatibility
         max_concurrent_requests=30 # number of concurrent requests with asyncIO
     )
     
@@ -127,7 +172,7 @@ async def process_files():
     config = LLMConfig(
         model_name="gpt-4o-mini",
         temperature=0.7,
-        max_tokens=200
+        max_completion_tokens=200
     )
     
     # Process all .txt files in a directory
@@ -183,7 +228,8 @@ Configuration class for LLM requests.
 LLMConfig(
     model_name: str,
     temperature: float = 0.7,
-    max_tokens: Optional[int] = None,
+    max_completion_tokens: Optional[int] = None,  # Preferred parameter
+    max_tokens: Optional[int] = None,  # Deprecated, kept for backward compatibility
     system_instruction: Optional[str] = None,
     max_retries: int = 10,
     max_concurrent_requests: int = 5,
@@ -199,7 +245,7 @@ Main function for batch processing of prompts.
 ```python
 async def process_prompts_batch(
     config: LLMConfig,
-    provider: str,  # "openai"
+    provider: str,  # "openai" or "together"
     prompts: Optional[List[str]] = None,
     input_dir: Optional[str] = None,
     cache_dir: str = "llm_cache",
@@ -256,6 +302,23 @@ llm_batch_helper/
 - gpt-4
 - gpt-3.5-turbo
 
+### Together.ai
+- meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+- meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
+- mistralai/Mixtral-8x7B-Instruct-v0.1
+- And many other open-source models
+
+## Documentation
+
+📖 **[Complete Documentation](https://llm-batch-helper.readthedocs.io/)** - Comprehensive docs on Read the Docs
+
+### Quick Links:
+- [Quick Start Guide](https://llm-batch-helper.readthedocs.io/en/latest/quickstart.html) - Get started quickly
+- [API Reference](https://llm-batch-helper.readthedocs.io/en/latest/api.html) - Complete API documentation  
+- [Examples](https://llm-batch-helper.readthedocs.io/en/latest/examples.html) - Practical usage examples
+- [Tutorials](https://llm-batch-helper.readthedocs.io/en/latest/tutorials.html) - Step-by-step tutorials
+- [Provider Guide](https://llm-batch-helper.readthedocs.io/en/latest/providers.html) - OpenAI & Together.ai setup
+
 ## Contributing
 
 1. Fork the repository
@@ -271,6 +334,12 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
+### v0.1.5
+- Added Together.ai provider support
+- Support for open-source models (Llama, Mixtral, etc.)
+- Enhanced documentation with Read the Docs
+- Updated examples and tutorials
+
 ### v0.1.0
 - Initial release
 - Support for OpenAI API

{llm_batch_helper-0.1.4 → llm_batch_helper-0.1.6}/README.md

@@ -1,13 +1,35 @@
 # LLM Batch Helper
 
-A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities and response caching.
+[![PyPI version](https://badge.fury.io/py/llm_batch_helper.svg)](https://badge.fury.io/py/llm_batch_helper)
+[![Downloads](https://pepy.tech/badge/llm_batch_helper)](https://pepy.tech/project/llm_batch_helper)
+[![Downloads/Month](https://pepy.tech/badge/llm_batch_helper/month)](https://pepy.tech/project/llm_batch_helper)
+[![Documentation Status](https://readthedocs.org/projects/llm-batch-helper/badge/?version=latest)](https://llm-batch-helper.readthedocs.io/en/latest/?badge=latest)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities, response caching, prompt verification, and more. This package is designed to streamline applications like LLM simulation, LLM-as-a-judge, and other batch processing scenarios.
+
+📖 **[Complete Documentation](https://llm-batch-helper.readthedocs.io/)** | 🚀 **[Quick Start Guide](https://llm-batch-helper.readthedocs.io/en/latest/quickstart.html)**
+
+## Why we designed this package
+
+Calling LLM APIs has become increasingly common, but several pain points exist in practice:
+
+1. **Efficient Batch Processing**: How do you run LLM calls in batches efficiently? Our async implementation is 3X-100X faster than multi-thread/multi-process approaches.
+
+2. **API Reliability**: LLM APIs can be unstable, so we need robust retry mechanisms when calls get interrupted.
+
+3. **Long-Running Simulations**: During long-running LLM simulations, computers can crash and APIs can fail. Can we cache LLM API calls to avoid repeating completed work?
+
+4. **Output Validation**: LLM outputs often have format requirements. If the output isn't right, we need to retry with validation.
+
+This package is designed to solve these exact pain points with async processing, intelligent caching, and comprehensive error handling. If there are some additional features you need, please post an issue.  
 
 ## Features
 
 - **Async Processing**: Submit multiple prompts concurrently for faster processing
 - **Response Caching**: Automatically cache responses to avoid redundant API calls
 - **Multiple Input Formats**: Support for both file-based and list-based prompts
-- **Provider Support**: Works with OpenAI API
+- **Provider Support**: Works with OpenAI and Together.ai APIs
 - **Retry Logic**: Built-in retry mechanism with exponential backoff
 - **Verification Callbacks**: Custom verification for response quality
 - **Progress Tracking**: Real-time progress bars for batch operations
@@ -39,9 +61,29 @@ poetry shell
 
 ### 1. Set up environment variables
 
+**Option A: Environment Variables**
 ```bash
 # For OpenAI
 export OPENAI_API_KEY="your-openai-api-key"
+
+# For Together.ai
+export TOGETHER_API_KEY="your-together-api-key"
+```
+
+**Option B: .env File (Recommended for Development)**
+```python
+# In your script, before importing llm_batch_helper
+from dotenv import load_dotenv
+load_dotenv()  # Load from .env file
+
+# Then use the package normally
+from llm_batch_helper import LLMConfig, process_prompts_batch
+```
+
+Create a `.env` file in your project:
+```
+OPENAI_API_KEY=your-openai-api-key
+TOGETHER_API_KEY=your-together-api-key
 ```
 
 ### 2. Interactive Tutorial (Recommended)
@@ -54,14 +96,18 @@ The tutorial covers all features with interactive examples!
 
 ```python
 import asyncio
+from dotenv import load_dotenv  # Optional: for .env file support
 from llm_batch_helper import LLMConfig, process_prompts_batch
 
+# Optional: Load environment variables from .env file
+load_dotenv()
+
 async def main():
     # Create configuration
     config = LLMConfig(
         model_name="gpt-4o-mini",
         temperature=0.7,
-        max_tokens=100,
+        max_completion_tokens=100,  # or use max_tokens for backward compatibility
         max_concurrent_requests=30 # number of concurrent requests with asyncIO
     )
     
@@ -99,7 +145,7 @@ async def process_files():
     config = LLMConfig(
         model_name="gpt-4o-mini",
         temperature=0.7,
-        max_tokens=200
+        max_completion_tokens=200
     )
     
     # Process all .txt files in a directory
@@ -155,7 +201,8 @@ Configuration class for LLM requests.
 LLMConfig(
     model_name: str,
     temperature: float = 0.7,
-    max_tokens: Optional[int] = None,
+    max_completion_tokens: Optional[int] = None,  # Preferred parameter
+    max_tokens: Optional[int] = None,  # Deprecated, kept for backward compatibility
     system_instruction: Optional[str] = None,
     max_retries: int = 10,
     max_concurrent_requests: int = 5,
@@ -171,7 +218,7 @@ Main function for batch processing of prompts.
 ```python
 async def process_prompts_batch(
     config: LLMConfig,
-    provider: str,  # "openai"
+    provider: str,  # "openai" or "together"
     prompts: Optional[List[str]] = None,
     input_dir: Optional[str] = None,
     cache_dir: str = "llm_cache",
@@ -228,6 +275,23 @@ llm_batch_helper/
 - gpt-4
 - gpt-3.5-turbo
 
+### Together.ai
+- meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+- meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
+- mistralai/Mixtral-8x7B-Instruct-v0.1
+- And many other open-source models
+
+## Documentation
+
+📖 **[Complete Documentation](https://llm-batch-helper.readthedocs.io/)** - Comprehensive docs on Read the Docs
+
+### Quick Links:
+- [Quick Start Guide](https://llm-batch-helper.readthedocs.io/en/latest/quickstart.html) - Get started quickly
+- [API Reference](https://llm-batch-helper.readthedocs.io/en/latest/api.html) - Complete API documentation  
+- [Examples](https://llm-batch-helper.readthedocs.io/en/latest/examples.html) - Practical usage examples
+- [Tutorials](https://llm-batch-helper.readthedocs.io/en/latest/tutorials.html) - Step-by-step tutorials
+- [Provider Guide](https://llm-batch-helper.readthedocs.io/en/latest/providers.html) - OpenAI & Together.ai setup
+
 ## Contributing
 
 1. Fork the repository
@@ -243,6 +307,12 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
+### v0.1.5
+- Added Together.ai provider support
+- Support for open-source models (Llama, Mixtral, etc.)
+- Enhanced documentation with Read the Docs
+- Updated examples and tutorials
+
 ### v0.1.0
 - Initial release
 - Support for OpenAI API

{llm_batch_helper-0.1.4 → llm_batch_helper-0.1.6}/llm_batch_helper/__init__.py

@@ -3,7 +3,7 @@ from .config import LLMConfig
 from .input_handlers import get_prompts, read_prompt_files, read_prompt_list
 from .providers import process_prompts_batch
 
-__version__ = "0.1.1"
+__version__ = "0.1.6"
 
 __all__ = [
     "LLMCache",

{llm_batch_helper-0.1.4 → llm_batch_helper-0.1.6}/llm_batch_helper/providers.py

@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 
 import httpx
 import openai
-from dotenv import load_dotenv
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 from tqdm.asyncio import tqdm_asyncio
 
@@ -12,8 +11,6 @@ from .cache import LLMCache
 from .config import LLMConfig
 from .input_handlers import get_prompts
 
-load_dotenv()
-
 
 @retry(
     stop=stop_after_attempt(5),
@@ -60,6 +57,65 @@ async def _get_openai_response_direct(
             "usage_details": usage_details,
         }
 
+
+@retry(
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(multiplier=1, min=4, max=60),
+    retry=retry_if_exception_type(
+        (
+            ConnectionError,
+            TimeoutError,
+            httpx.HTTPStatusError,
+            httpx.RequestError,
+        )
+    ),
+    reraise=True,
+)
+async def _get_together_response_direct(
+    prompt: str, config: LLMConfig
+) -> Dict[str, Union[str, Dict]]:
+    api_key = os.environ.get("TOGETHER_API_KEY")
+    if not api_key:
+        raise ValueError("TOGETHER_API_KEY environment variable not set")
+
+    async with httpx.AsyncClient(timeout=1000.0) as client:
+        messages = [
+            {"role": "system", "content": config.system_instruction},
+            {"role": "user", "content": prompt},
+        ]
+
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+
+        payload = {
+            "model": config.model_name,
+            "messages": messages,
+            "temperature": config.temperature,
+            "max_tokens": config.max_completion_tokens,
+        }
+
+        response = await client.post(
+            "https://api.together.xyz/chat/completions",
+            json=payload,
+            headers=headers,
+        )
+        response.raise_for_status()
+        
+        response_data = response.json()
+        usage = response_data.get("usage", {})
+        usage_details = {
+            "prompt_token_count": usage.get("prompt_tokens", 0),
+            "completion_token_count": usage.get("completion_tokens", 0),
+            "total_token_count": usage.get("total_tokens", 0),
+        }
+        
+        return {
+            "response_text": response_data["choices"][0]["message"]["content"],
+            "usage_details": usage_details,
+        }
+
 async def get_llm_response_with_internal_retry(
     prompt_id: str,
     prompt: str,
@@ -77,6 +133,8 @@ async def get_llm_response_with_internal_retry(
     try:
         if provider.lower() == "openai":
             response = await _get_openai_response_direct(prompt, config)
+        elif provider.lower() == "together":
+            response = await _get_together_response_direct(prompt, config)
         else:
             raise ValueError(f"Unsupported provider: {provider}")
 
@@ -107,7 +165,7 @@ async def process_prompts_batch(
         prompts: Optional list of prompts in any supported format (string, tuple, or dict)
         input_dir: Optional path to directory containing prompt files
         config: LLM configuration
-        provider: LLM provider to use ("openai" or "gemini")
+        provider: LLM provider to use ("openai", "together", or "gemini")
         desc: Description for progress bar
         cache_dir: Optional directory for caching responses
         force: If True, force regeneration even if cached response exists

{llm_batch_helper-0.1.4 → llm_batch_helper-0.1.6}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "llm_batch_helper"
-version = "0.1.4"
+version = "0.1.6"
 description = "A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities and response caching."
 authors = ["Tianyi Peng <tianyipeng95@gmail.com>"]
 readme = "README.md"
 license = "MIT"
 homepage = "https://github.com/TianyiPeng/LLM_batch_helper"
 repository = "https://github.com/TianyiPeng/LLM_batch_helper"
-keywords = ["llm", "openai", "batch", "async", "ai", "nlp", "api"]
+keywords = ["llm", "openai", "together", "batch", "async", "ai", "nlp", "api"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
@@ -25,11 +25,11 @@ packages = [{include = "llm_batch_helper"}]
 python = "^3.11"
 httpx = ">=0.24.0,<2.0.0"
 openai = "^1.0.0"
-python-dotenv = "^1.0.0"
 tenacity = "^8.0.0"
 tqdm = "^4.65.0"
 
 [tool.poetry.group.dev.dependencies]
+python-dotenv = "^1.0.0"  # Optional for .env file support
 pytest = "^7.0.0"
 black = "^23.0.0"
 isort = "^5.12.0"
@@ -39,6 +39,10 @@ jupyter = "^1.0.0"
 twine = "^6.1.0"
 ipython = "^9.4.0"
 ipykernel = "^6.29.5"
+sphinx = "^7.0.0"
+sphinx-rtd-theme = "^2.0.0"
+myst-parser = "^2.0.0"
+sphinx-autodoc-typehints = "^1.24.0"
 
 [build-system]
 requires = ["poetry-core"]