llm_batch_helper 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

llm_batch_helper/__init__.py

@@ -1,13 +1,16 @@
 from .cache import LLMCache
 from .config import LLMConfig
+from .exceptions import InvalidPromptFormatError, VerificationFailedError
 from .input_handlers import get_prompts, read_prompt_files, read_prompt_list
 from .providers import process_prompts_batch, process_prompts_batch_async
 
-__version__ = "0.3.1"
+__version__ = "0.3.3"
 
 __all__ = [
     "LLMCache",
     "LLMConfig",
+    "InvalidPromptFormatError",
+    "VerificationFailedError",
     "get_prompts",
     "process_prompts_batch",
     "process_prompts_batch_async",  # For backward compatibility

llm_batch_helper/exceptions.py

@@ -5,3 +5,11 @@ class VerificationFailedError(Exception):
         super().__init__(message)
         self.prompt_id = prompt_id
         self.llm_response_data = llm_response_data
+
+
+class InvalidPromptFormatError(Exception):
+    """Custom exception for invalid prompt format."""
+
+    def __init__(self, message, invalid_item=None):
+        super().__init__(message)
+        self.invalid_item = invalid_item

llm_batch_helper/input_handlers.py

@@ -2,6 +2,8 @@ import hashlib
 from pathlib import Path
 from typing import Any, Dict, List, Tuple, Union
 
+from .exceptions import InvalidPromptFormatError
+
 
 def read_prompt_files(input_dir: str) -> List[Tuple[str, str]]:
     """Read all text files from input directory and return as (filename, content) pairs.
@@ -53,12 +55,34 @@ def read_prompt_list(
         elif isinstance(item, tuple) and len(item) == 2:
             # Tuple format: (prompt_id, prompt_text)
             prompt_id, prompt_text = item
-        elif isinstance(item, dict) and "id" in item and "text" in item:
-            # Dict format: {"id": prompt_id, "text": prompt_text}
+        elif isinstance(item, dict):
+            # Dict format: must have both "id" and "text" keys
+            if "id" not in item:
+                raise InvalidPromptFormatError(
+                    f"Dictionary prompt is missing required 'id' key. "
+                    f"Dictionary format must be: {{'id': 'prompt_id', 'text': 'prompt_text'}}. "
+                    f"Got: {item}",
+                    invalid_item=item
+                )
+            if "text" not in item:
+                raise InvalidPromptFormatError(
+                    f"Dictionary prompt is missing required 'text' key. "
+                    f"Dictionary format must be: {{'id': 'prompt_id', 'text': 'prompt_text'}}. "
+                    f"Got: {item}",
+                    invalid_item=item
+                )
             prompt_id = item["id"]
             prompt_text = item["text"]
         else:
-            raise ValueError(f"Invalid prompt format: {item}")
+            raise InvalidPromptFormatError(
+                f"Invalid prompt format. Expected str, tuple, or dict, got {type(item).__name__}. "
+                f"Valid formats: "
+                f"- str: 'prompt text' "
+                f"- tuple: ('prompt_id', 'prompt_text') "
+                f"- dict: {{'id': 'prompt_id', 'text': 'prompt_text'}}. "
+                f"Got: {item}",
+                invalid_item=item
+            )
         prompts.append((prompt_id, prompt_text))
     return prompts
 

llm_batch_helper/providers.py

@@ -288,10 +288,11 @@ async def process_prompts_batch_async(
         force: If True, force regeneration even if cached response exists
 
     Returns:
-        Dict mapping prompt IDs to their responses
+        Dict mapping prompt IDs to their responses, ordered by input sequence
 
     Note:
         Either prompts or input_dir must be provided, but not both.
+        Results are returned in the same order as the input prompts.
     """
     if prompts is None and input_dir is None:
         raise ValueError("Either prompts or input_dir must be provided")
@@ -309,6 +310,9 @@ async def process_prompts_batch_async(
 
     # Process prompts
     results = {}
+    # Keep track of original order for sorting results
+    prompt_order = {prompt_id: idx for idx, (prompt_id, _) in enumerate(prompts)}
+    
     tasks = [
         _process_single_prompt_attempt_with_verification(
             prompt_id, prompt_text, config, provider, semaphore, cache_dir, force
@@ -320,7 +324,14 @@ async def process_prompts_batch_async(
         prompt_id, response_data = await future
         results[prompt_id] = response_data
 
-    return results
+    # Sort results by original input order to maintain input sequence
+    # Note: Python 3.7+ guarantees dict insertion order, we explicitly sort
+    # to ensure results match the original prompt order regardless of completion order
+    ordered_results = {}
+    for prompt_id in sorted(results.keys(), key=lambda pid: prompt_order[pid]):
+        ordered_results[prompt_id] = results[prompt_id]
+    
+    return ordered_results
 
 
 def process_prompts_batch(
@@ -348,10 +359,11 @@ def process_prompts_batch(
         force: If True, force regeneration even if cached response exists
 
     Returns:
-        Dict mapping prompt IDs to their responses
+        Dict mapping prompt IDs to their responses, ordered by input sequence
 
     Note:
         Either prompts or input_dir must be provided, but not both.
+        Results are returned in the same order as the input prompts.
         
     Example:
         >>> from llm_batch_helper import LLMConfig, process_prompts_batch
@@ -361,6 +373,7 @@ def process_prompts_batch(
         ...     config=config,
         ...     provider="openai"
         ... )
+        >>> # Results will be in the same order as input prompts
     """
     return _run_async_function(
         process_prompts_batch_async,
@@ -390,18 +403,22 @@ async def _process_single_prompt_attempt_with_verification(
             cache = LLMCache(cache_dir)
             cached_response = cache.get_cached_response(prompt_id)
             if cached_response is not None:
-                # Verify response if callback provided
                 cached_response_data = cached_response["llm_response"]
-                if config.verification_callback:
-                    verified = await asyncio.to_thread(
-                        config.verification_callback,
-                        prompt_id,
-                        cached_response_data,
-                        prompt_text,
-                        **config.verification_callback_args,
-                    )
-                    if verified:
-                        return prompt_id, {**cached_response_data, "from_cache": True}
+                
+                # If no verification callback, use cached response directly
+                if config.verification_callback is None:
+                    return prompt_id, {**cached_response_data, "from_cache": True}
+                
+                # Verify response if callback provided
+                verified = await asyncio.to_thread(
+                    config.verification_callback,
+                    prompt_id,
+                    cached_response_data,
+                    prompt_text,
+                    **config.verification_callback_args,
+                )
+                if verified:
+                    return prompt_id, {**cached_response_data, "from_cache": True}
 
         # Process the prompt
         last_exception_details = None

{llm_batch_helper-0.3.1.dist-info → llm_batch_helper-0.3.3.dist-info}/METADATA

@@ -1,20 +1,20 @@
 Metadata-Version: 2.3
 Name: llm_batch_helper
-Version: 0.3.1
+Version: 0.3.3
 Summary: A Python package that enables batch submission of prompts to LLM APIs, with simplified interface and built-in async capabilities handled implicitly.
 License: MIT
 Keywords: llm,openai,together,openrouter,batch,async,ai,nlp,api
 Author: Tianyi Peng
 Author-email: tianyipeng95@gmail.com
-Requires-Python: >=3.11,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: httpx (>=0.24.0,<2.0.0)
@@ -39,9 +39,9 @@ A Python package that enables batch submission of prompts to LLM APIs, with buil
 
 ## Why we designed this package
 
-Calling LLM APIs has become increasingly common, but several pain points exist in practice:
+Imagine you have 5000 prompts you need to send to an LLM. Running them sequentially can be painfully slow—sometimes taking hours or even days. Worse, if the process fails midway, you’re forced to start all over again. We’ve struggled with this exact frustration, which is why we built this package, to directly tackle these pain points:
 
-1. **Efficient Batch Processing**: How do you run LLM calls in batches efficiently? Our async implementation is 3X-100X faster than multi-thread/multi-process approaches.
+1. **Efficient Batch Processing**: How do you run LLM calls in batches efficiently? Our async implementation is 3X-100X faster than multi-thread/multi-process approaches. In my own experience, it reduces the time from 24 hours to 10min. 
 
 2. **API Reliability**: LLM APIs can be unstable, so we need robust retry mechanisms when calls get interrupted.
 
@@ -53,46 +53,31 @@ This package is designed to solve these exact pain points with async processing,
 
 ## Features
 
-- **Async Processing**: Submit multiple prompts concurrently for faster processing
-- **Response Caching**: Automatically cache responses to avoid redundant API calls
-- **Multiple Input Formats**: Support for both file-based and list-based prompts
-- **Provider Support**: Works with OpenAI (all models including GPT-5), OpenRouter (100+ models), and Together.ai APIs
-- **Retry Logic**: Built-in retry mechanism with exponential backoff and detailed logging
-- **Verification Callbacks**: Custom verification for response quality  
-- **Progress Tracking**: Real-time progress bars for batch operations
-- **Simplified API**: Async operations handled implicitly - no async/await needed (v0.3.0+)
-- **Detailed Error Logging**: See exactly what happens during retries with timestamps and error details
+- **🚀 Dramatic Speed Improvements**: **10-100x faster** than sequential processing ([see demo](https://github.com/TianyiPeng/LLM_batch_helper/blob/main/tutorials/performance_comparison_tutorial.ipynb))
+- **⚡ Async Processing**: Submit multiple prompts concurrently for maximum throughput
+- **💾 Smart Caching**: Automatically cache responses and resume interrupted work seamlessly
+- **📝 Multiple Input Formats**: Support for strings, tuples, dictionaries, and file-based prompts
+- **🌐 Multi-Provider Support**: Works with OpenAI (all models), OpenRouter (100+ models), and Together.ai
+- **🔄 Intelligent Retry Logic**: Built-in retry mechanism with exponential backoff and detailed logging
+- **✅ Quality Control**: Custom verification callbacks for response validation
+- **📊 Progress Tracking**: Real-time progress bars and comprehensive statistics
+- **🎯 Simplified API**: No async/await complexity - works seamlessly in Jupyter notebooks (v0.3.0+)
+- **🔧 Tunable Performance**: Adjust concurrency on-the-fly for optimal speed vs rate limits
 
 ## Installation
 
-### For Users (Recommended)
-
 ```bash
 # Install from PyPI
 pip install llm_batch_helper
 ```
 
-### For Development
-
-```bash
-# Clone the repository
-git clone https://github.com/TianyiPeng/LLM_batch_helper.git
-cd llm_batch_helper
-
-# Install with Poetry
-poetry install
-
-# Activate the virtual environment
-poetry shell
-```
-
 ## Quick Start
 
 ### 1. Set up environment variables
 
 **Option A: Environment Variables**
 ```bash
-# For OpenAI (all models including GPT-5)
+# For OpenAI (all OpenAI models including GPT-5)
 export OPENAI_API_KEY="your-openai-api-key"
 
 # For OpenRouter (100+ models - Recommended)
@@ -103,6 +88,11 @@ export TOGETHER_API_KEY="your-together-api-key"
 ```
 
 **Option B: .env File (Recommended for Development)**
+Create a `.env` file in your project:
+```
+OPENAI_API_KEY=your-openai-api-key
+```
+
 ```python
 # In your script, before importing llm_batch_helper
 from dotenv import load_dotenv
@@ -112,17 +102,17 @@ load_dotenv()  # Load from .env file
 from llm_batch_helper import LLMConfig, process_prompts_batch
 ```
 
-Create a `.env` file in your project:
-```
-OPENAI_API_KEY=your-openai-api-key
-TOGETHER_API_KEY=your-together-api-key
-```
-
-### 2. Interactive Tutorial (Recommended)
+### 2. Interactive Tutorials (Recommended)
 
-Check out the comprehensive Jupyter notebook [tutorial](https://github.com/TianyiPeng/LLM_batch_helper/blob/main/tutorials/llm_batch_helper_tutorial.ipynb).
+**🎯 NEW: Performance Comparison Tutorial**
+See the dramatic speed improvements! Our [Performance Comparison Tutorial](https://github.com/TianyiPeng/LLM_batch_helper/blob/main/tutorials/performance_comparison_tutorial.ipynb) demonstrates:
+- **10-100x speedup** vs naive sequential processing
+- Processing **5,000 prompts** in minutes instead of hours
+- **Smart caching** that lets you resume interrupted work
+- **Tunable concurrency** for optimal performance
 
-The tutorial covers all features with interactive examples!
+**📚 Complete Feature Tutorial**
+Check out the comprehensive [main tutorial](https://github.com/TianyiPeng/LLM_batch_helper/blob/main/tutorials/llm_batch_helper_tutorial.ipynb) covering all features with interactive examples!
 
 ### 3. Basic usage
 
@@ -138,10 +128,10 @@ config = LLMConfig(
     model_name="gpt-4o-mini",
     temperature=1.0,
     max_completion_tokens=100,
-    max_concurrent_requests=30  # number of concurrent requests with asyncIO
+    max_concurrent_requests=100  # number of concurrent requests with asyncIO, this number decides how fast your pipeline can run. We suggest a number that is as large as possible (e.g., 300) while making sure you are not over the rate limit constrained by the LLM APIs. 
 )
 
-# Process prompts - no async/await needed!
+# Process prompts
 prompts = [
     "What is the capital of France?",
     "What is 2+2?",
@@ -162,6 +152,49 @@ for prompt_id, response in results.items():
 
 **🎉 New in v0.3.0**: `process_prompts_batch` now handles async operations **implicitly** - no more async/await syntax needed! Works seamlessly in Jupyter notebooks.
 
+### 4. Multiple Input Formats
+
+The package supports three different input formats for maximum flexibility:
+
+```python
+from llm_batch_helper import LLMConfig, process_prompts_batch
+
+config = LLMConfig(
+    model_name="gpt-4o-mini",
+    temperature=1.0,
+    max_completion_tokens=100
+)
+
+# Mix different input formats in the same list
+prompts = [
+    # String format - ID will be auto-generated from hash
+    "What is the capital of France?",
+    
+    # Tuple format - (custom_id, prompt_text)
+    ("custom_id_1", "What is 2+2?"),
+    
+    # Dictionary format - {"id": custom_id, "text": prompt_text}
+    {"id": "shakespeare_q", "text": "Who wrote 'Hamlet'?"},
+    {"id": "science_q", "text": "Explain photosynthesis briefly."}
+]
+
+results = process_prompts_batch(
+    config=config,
+    provider="openai",
+    prompts=prompts,
+    cache_dir="cache"
+)
+
+# Print results with custom IDs
+for prompt_id, response in results.items():
+    print(f"{prompt_id}: {response['response_text']}")
+```
+
+**Input Format Requirements:**
+- **String**: Plain text prompt (ID auto-generated)
+- **Tuple**: `(prompt_id, prompt_text)` - both elements required
+- **Dictionary**: `{"id": "prompt_id", "text": "prompt_text"}` - both keys required
+
 ### 🔄 Backward Compatibility
 
 For users who prefer the async version or have existing code, the async API is still available:
@@ -352,7 +385,8 @@ llm_batch_helper/
 │   ├── prompts/               # Sample prompt files
 │   └── llm_cache/             # Example cache directory
 └── tutorials/                 # Interactive tutorials
-    └── llm_batch_helper_tutorial.ipynb  # Comprehensive Jupyter notebook tutorial
+    ├── llm_batch_helper_tutorial.ipynb  # Comprehensive feature tutorial
+    └── performance_comparison_tutorial.ipynb  # Performance demo (NEW!)
 ```
 
 ## Supported Models

llm_batch_helper-0.3.3.dist-info/RECORD

@@ -0,0 +1,10 @@
+llm_batch_helper/__init__.py,sha256=dFEq-gZrOkRpvRWib8RiAP9h9Ww9NrRVGe4lLAxH7H0,579
+llm_batch_helper/cache.py,sha256=QUODQ1tPCvFThO3yvVOTcorcOrmN2dP5HLF1Y2O1bTQ,1276
+llm_batch_helper/config.py,sha256=vM3YEeLYkN9qfY1fTPXN27wAlTeaVXXCfcFw7nvbIqw,1372
+llm_batch_helper/exceptions.py,sha256=FvcBV5gpSAmk_XwoBecGGqFeVDTawgeV6txw6Tob-z0,523
+llm_batch_helper/input_handlers.py,sha256=KeEwWQZc6c9QYQwOCPEVb_njM5WVZF3d2Staomd2KjA,3945
+llm_batch_helper/providers.py,sha256=o0gGp9uHOVKJQ8ZK2KYHDocjqPpiABMisVApvG35eaM,17765
+llm_batch_helper-0.3.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+llm_batch_helper-0.3.3.dist-info/METADATA,sha256=nRaotV4SD8v6RufE4JKDGuRPZnBMrf0wJtEdfefJzi4,16740
+llm_batch_helper-0.3.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+llm_batch_helper-0.3.3.dist-info/RECORD,,

llm_batch_helper-0.3.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-llm_batch_helper/__init__.py,sha256=sCQ-eJVy3cCs80ASvcww10om_E9CHDctCfpIGLPIcy8,442
-llm_batch_helper/cache.py,sha256=QUODQ1tPCvFThO3yvVOTcorcOrmN2dP5HLF1Y2O1bTQ,1276
-llm_batch_helper/config.py,sha256=vM3YEeLYkN9qfY1fTPXN27wAlTeaVXXCfcFw7nvbIqw,1372
-llm_batch_helper/exceptions.py,sha256=59_f3jINUhKFble6HTp8pmtLSFE2MYLHWGclwaQKs28,296
-llm_batch_helper/input_handlers.py,sha256=IadA732F1Rw0zcBok5hjZr32RUm8eTUOpvLsRuMvaE4,2877
-llm_batch_helper/providers.py,sha256=zv7dCiKZtSOcdV-4kvd3WKhClOv1jio9neZqGcYskm8,16794
-llm_batch_helper-0.3.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-llm_batch_helper-0.3.1.dist-info/METADATA,sha256=AAzYnTi_1DhVn3subd9XT3LLFbKP63MCn1cptZTgMAc,14505
-llm_batch_helper-0.3.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-llm_batch_helper-0.3.1.dist-info/RECORD,,