llm_batch_helper 0.1.6__tar.gz → 0.3.0__tar.gz

{llm_batch_helper-0.1.6 → llm_batch_helper-0.3.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: llm_batch_helper
-Version: 0.1.6
-Summary: A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities and response caching.
+Version: 0.3.0
+Summary: A Python package that enables batch submission of prompts to LLM APIs, with simplified interface and built-in async capabilities handled implicitly.
 License: MIT
-Keywords: llm,openai,together,batch,async,ai,nlp,api
+Keywords: llm,openai,together,openrouter,batch,async,ai,nlp,api
 Author: Tianyi Peng
 Author-email: tianyipeng95@gmail.com
 Requires-Python: >=3.11,<4.0
@@ -56,10 +56,12 @@ This package is designed to solve these exact pain points with async processing,
 - **Async Processing**: Submit multiple prompts concurrently for faster processing
 - **Response Caching**: Automatically cache responses to avoid redundant API calls
 - **Multiple Input Formats**: Support for both file-based and list-based prompts
-- **Provider Support**: Works with OpenAI and Together.ai APIs
-- **Retry Logic**: Built-in retry mechanism with exponential backoff
-- **Verification Callbacks**: Custom verification for response quality
+- **Provider Support**: Works with OpenAI (all models including GPT-5), OpenRouter (100+ models), and Together.ai APIs
+- **Retry Logic**: Built-in retry mechanism with exponential backoff and detailed logging
+- **Verification Callbacks**: Custom verification for response quality  
 - **Progress Tracking**: Real-time progress bars for batch operations
+- **Simplified API**: Async operations handled implicitly - no async/await needed (v0.3.0+)
+- **Detailed Error Logging**: See exactly what happens during retries with timestamps and error details
 
 ## Installation
 
@@ -90,9 +92,12 @@ poetry shell
 
 **Option A: Environment Variables**
 ```bash
-# For OpenAI
+# For OpenAI (all models including GPT-5)
 export OPENAI_API_KEY="your-openai-api-key"
 
+# For OpenRouter (100+ models - Recommended)
+export OPENROUTER_API_KEY="your-openrouter-api-key"
+
 # For Together.ai
 export TOGETHER_API_KEY="your-together-api-key"
 ```
@@ -122,71 +127,111 @@ The tutorial covers all features with interactive examples!
 ### 3. Basic usage
 
 ```python
-import asyncio
 from dotenv import load_dotenv  # Optional: for .env file support
 from llm_batch_helper import LLMConfig, process_prompts_batch
 
 # Optional: Load environment variables from .env file
 load_dotenv()
 
+# Create configuration
+config = LLMConfig(
+    model_name="gpt-4o-mini",
+    temperature=1.0,
+    max_completion_tokens=100,
+    max_concurrent_requests=30  # number of concurrent requests with asyncIO
+)
+
+# Process prompts - no async/await needed!
+prompts = [
+    "What is the capital of France?",
+    "What is 2+2?",
+    "Who wrote 'Hamlet'?"
+]
+
+results = process_prompts_batch(
+    config=config,
+    provider="openai",
+    prompts=prompts,
+    cache_dir="cache"
+)
+
+# Print results
+for prompt_id, response in results.items():
+    print(f"{prompt_id}: {response['response_text']}")
+```
+
+**🎉 New in v0.3.0**: `process_prompts_batch` now handles async operations **implicitly** - no more async/await syntax needed! Works seamlessly in Jupyter notebooks.
+
+### 🔄 Backward Compatibility
+
+For users who prefer the async version or have existing code, the async API is still available:
+
+```python
+import asyncio
+from llm_batch_helper import process_prompts_batch_async
+
 async def main():
-    # Create configuration
-    config = LLMConfig(
-        model_name="gpt-4o-mini",
-        temperature=0.7,
-        max_completion_tokens=100,  # or use max_tokens for backward compatibility
-        max_concurrent_requests=30 # number of concurrent requests with asyncIO
-    )
-    
-    # Process prompts
-    prompts = [
-        "What is the capital of France?",
-        "What is 2+2?",
-        "Who wrote 'Hamlet'?"
-    ]
-    
-    results = await process_prompts_batch(
+    results = await process_prompts_batch_async(
+        prompts=["Hello world!"],
         config=config,
-        provider="openai",
-        prompts=prompts,
-        cache_dir="cache"
+        provider="openai"
     )
-    
-    # Print results
-    for prompt_id, response in results.items():
-        print(f"{prompt_id}: {response['response_text']}")
+    return results
 
-if __name__ == "__main__":
-    asyncio.run(main())
+results = asyncio.run(main())
 ```
 
 ## Usage Examples
 
+### OpenRouter (Recommended - 100+ Models)
+
+```python
+from llm_batch_helper import LLMConfig, process_prompts_batch
+
+# Access 100+ models through OpenRouter
+config = LLMConfig(
+    model_name="deepseek/deepseek-v3.1-base",  # or openai/gpt-4o, anthropic/claude-3-5-sonnet
+    temperature=1.0,
+    max_completion_tokens=500
+)
+
+prompts = [
+    "Explain quantum computing briefly.",
+    "What are the benefits of renewable energy?",
+    "How does machine learning work?"
+]
+
+results = process_prompts_batch(
+    prompts=prompts,
+    config=config,
+    provider="openrouter"  # Access to 100+ models!
+)
+
+for prompt_id, result in results.items():
+    print(f"Response: {result['response_text']}")
+```
+
 ### File-based Prompts
 
 ```python
-import asyncio
 from llm_batch_helper import LLMConfig, process_prompts_batch
 
-async def process_files():
-    config = LLMConfig(
-        model_name="gpt-4o-mini",
-        temperature=0.7,
-        max_completion_tokens=200
-    )
-    
-    # Process all .txt files in a directory
-    results = await process_prompts_batch(
-        config=config,
-        provider="openai",
-        input_dir="prompts",  # Directory containing .txt files
-        cache_dir="cache",
-        force=False  # Use cached responses if available
-    )
-    
-    return results
+config = LLMConfig(
+    model_name="gpt-4o-mini",
+    temperature=1.0,
+    max_completion_tokens=200
+)
+
+# Process all .txt files in a directory
+results = process_prompts_batch(
+    config=config,
+    provider="openai",
+    input_dir="prompts",  # Directory containing .txt files
+    cache_dir="cache",
+    force=False  # Use cached responses if available
+)
 
-asyncio.run(process_files())
+print(f"Processed {len(results)} prompts from files")
 ```
 
 ### Custom Verification
@@ -210,7 +255,7 @@ def verify_response(prompt_id, llm_response_data, original_prompt_text, **kwargs
 
 config = LLMConfig(
     model_name="gpt-4o-mini",
-    temperature=0.7,
+    temperature=1.0,
     verification_callback=verify_response,
     verification_callback_args={"min_length": 20}
 )
@@ -227,7 +272,7 @@ Configuration class for LLM requests.
 ```python
 LLMConfig(
     model_name: str,
-    temperature: float = 0.7,
+    temperature: float = 1.0,
     max_completion_tokens: Optional[int] = None,  # Preferred parameter
     max_tokens: Optional[int] = None,  # Deprecated, kept for backward compatibility
     system_instruction: Optional[str] = None,
@@ -240,12 +285,28 @@ LLMConfig(
 
 ### process_prompts_batch
 
-Main function for batch processing of prompts.
+Main function for batch processing of prompts (async operations handled implicitly).
 
 ```python
-async def process_prompts_batch(
+def process_prompts_batch(
     config: LLMConfig,
-    provider: str,  # "openai" or "together"
+    provider: str,  # "openai", "openrouter" (recommended), or "together"
+    prompts: Optional[List[str]] = None,
+    input_dir: Optional[str] = None,
+    cache_dir: str = "llm_cache",
+    force: bool = False,
+    desc: str = "Processing prompts"
+) -> Dict[str, Dict[str, Any]]
+```
+
+### process_prompts_batch_async
+
+Async version for backward compatibility and advanced use cases.
+
+```python
+async def process_prompts_batch_async(
+    config: LLMConfig,
+    provider: str,  # "openai", "openrouter" (recommended), or "together"
     prompts: Optional[List[str]] = None,
     input_dir: Optional[str] = None,
     cache_dir: str = "llm_cache",
@@ -297,10 +358,15 @@ llm_batch_helper/
 ## Supported Models
 
 ### OpenAI
-- gpt-4o-mini
-- gpt-4o
-- gpt-4
-- gpt-3.5-turbo
+- **All OpenAI models** 
+
+### OpenRouter (Recommended - 100+ Models)
+- **OpenAI models**: `openai/gpt-4o`, `openai/gpt-4o-mini`
+- **Anthropic models**: `anthropic/claude-3-5-sonnet`, `anthropic/claude-3-haiku`
+- **DeepSeek models**: `deepseek/deepseek-v3.1-base`, `deepseek/deepseek-chat`
+- **Meta models**: `meta-llama/llama-3.1-405b-instruct`
+- **Google models**: `google/gemini-pro-1.5`
+- **And 90+ more models** from all major providers
 
 ### Together.ai
 - meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
@@ -317,7 +383,7 @@ llm_batch_helper/
 - [API Reference](https://llm-batch-helper.readthedocs.io/en/latest/api.html) - Complete API documentation  
 - [Examples](https://llm-batch-helper.readthedocs.io/en/latest/examples.html) - Practical usage examples
 - [Tutorials](https://llm-batch-helper.readthedocs.io/en/latest/tutorials.html) - Step-by-step tutorials
-- [Provider Guide](https://llm-batch-helper.readthedocs.io/en/latest/providers.html) - OpenAI & Together.ai setup
+- [Provider Guide](https://llm-batch-helper.readthedocs.io/en/latest/providers.html) - OpenAI, OpenRouter & Together.ai setup
 
 ## Contributing
 
@@ -334,6 +400,19 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
+### v0.3.0
+- **🎉 Major Update**: Simplified API - async operations handled implicitly, no async/await required!
+- **📓 Jupyter Support**: Works seamlessly in notebooks without event loop issues
+- **🔍 Detailed Retry Logging**: See exactly what happens during retries with timestamps
+- **🔄 Backward Compatibility**: Original async API still available as `process_prompts_batch_async`
+- **📚 Updated Examples**: All documentation updated to show simplified usage
+- **⚡ Smart Event Loop Handling**: Automatically detects and handles different Python environments
+
+### v0.2.0
+- Enhanced API stability
+- Improved error handling
+- Better documentation
+
 ### v0.1.5
 - Added Together.ai provider support
 - Support for open-source models (Llama, Mixtral, etc.)

{llm_batch_helper-0.1.6 → llm_batch_helper-0.3.0}/README.md

@@ -29,10 +29,12 @@ This package is designed to solve these exact pain points with async processing,
 - **Async Processing**: Submit multiple prompts concurrently for faster processing
 - **Response Caching**: Automatically cache responses to avoid redundant API calls
 - **Multiple Input Formats**: Support for both file-based and list-based prompts
-- **Provider Support**: Works with OpenAI and Together.ai APIs
-- **Retry Logic**: Built-in retry mechanism with exponential backoff
-- **Verification Callbacks**: Custom verification for response quality
+- **Provider Support**: Works with OpenAI (all models including GPT-5), OpenRouter (100+ models), and Together.ai APIs
+- **Retry Logic**: Built-in retry mechanism with exponential backoff and detailed logging
+- **Verification Callbacks**: Custom verification for response quality  
 - **Progress Tracking**: Real-time progress bars for batch operations
+- **Simplified API**: Async operations handled implicitly - no async/await needed (v0.3.0+)
+- **Detailed Error Logging**: See exactly what happens during retries with timestamps and error details
 
 ## Installation
 
@@ -63,9 +65,12 @@ poetry shell
 
 **Option A: Environment Variables**
 ```bash
-# For OpenAI
+# For OpenAI (all models including GPT-5)
 export OPENAI_API_KEY="your-openai-api-key"
 
+# For OpenRouter (100+ models - Recommended)
+export OPENROUTER_API_KEY="your-openrouter-api-key"
+
 # For Together.ai
 export TOGETHER_API_KEY="your-together-api-key"
 ```
@@ -95,71 +100,111 @@ The tutorial covers all features with interactive examples!
 ### 3. Basic usage
 
 ```python
-import asyncio
 from dotenv import load_dotenv  # Optional: for .env file support
 from llm_batch_helper import LLMConfig, process_prompts_batch
 
 # Optional: Load environment variables from .env file
 load_dotenv()
 
+# Create configuration
+config = LLMConfig(
+    model_name="gpt-4o-mini",
+    temperature=1.0,
+    max_completion_tokens=100,
+    max_concurrent_requests=30  # number of concurrent requests with asyncIO
+)
+
+# Process prompts - no async/await needed!
+prompts = [
+    "What is the capital of France?",
+    "What is 2+2?",
+    "Who wrote 'Hamlet'?"
+]
+
+results = process_prompts_batch(
+    config=config,
+    provider="openai",
+    prompts=prompts,
+    cache_dir="cache"
+)
+
+# Print results
+for prompt_id, response in results.items():
+    print(f"{prompt_id}: {response['response_text']}")
+```
+
+**🎉 New in v0.3.0**: `process_prompts_batch` now handles async operations **implicitly** - no more async/await syntax needed! Works seamlessly in Jupyter notebooks.
+
+### 🔄 Backward Compatibility
+
+For users who prefer the async version or have existing code, the async API is still available:
+
+```python
+import asyncio
+from llm_batch_helper import process_prompts_batch_async
+
 async def main():
-    # Create configuration
-    config = LLMConfig(
-        model_name="gpt-4o-mini",
-        temperature=0.7,
-        max_completion_tokens=100,  # or use max_tokens for backward compatibility
-        max_concurrent_requests=30 # number of concurrent requests with asyncIO
-    )
-    
-    # Process prompts
-    prompts = [
-        "What is the capital of France?",
-        "What is 2+2?",
-        "Who wrote 'Hamlet'?"
-    ]
-    
-    results = await process_prompts_batch(
+    results = await process_prompts_batch_async(
+        prompts=["Hello world!"],
         config=config,
-        provider="openai",
-        prompts=prompts,
-        cache_dir="cache"
+        provider="openai"
     )
-    
-    # Print results
-    for prompt_id, response in results.items():
-        print(f"{prompt_id}: {response['response_text']}")
+    return results
 
-if __name__ == "__main__":
-    asyncio.run(main())
+results = asyncio.run(main())
 ```
 
 ## Usage Examples
 
+### OpenRouter (Recommended - 100+ Models)
+
+```python
+from llm_batch_helper import LLMConfig, process_prompts_batch
+
+# Access 100+ models through OpenRouter
+config = LLMConfig(
+    model_name="deepseek/deepseek-v3.1-base",  # or openai/gpt-4o, anthropic/claude-3-5-sonnet
+    temperature=1.0,
+    max_completion_tokens=500
+)
+
+prompts = [
+    "Explain quantum computing briefly.",
+    "What are the benefits of renewable energy?",
+    "How does machine learning work?"
+]
+
+results = process_prompts_batch(
+    prompts=prompts,
+    config=config,
+    provider="openrouter"  # Access to 100+ models!
+)
+
+for prompt_id, result in results.items():
+    print(f"Response: {result['response_text']}")
+```
+
 ### File-based Prompts
 
 ```python
-import asyncio
 from llm_batch_helper import LLMConfig, process_prompts_batch
 
-async def process_files():
-    config = LLMConfig(
-        model_name="gpt-4o-mini",
-        temperature=0.7,
-        max_completion_tokens=200
-    )
-    
-    # Process all .txt files in a directory
-    results = await process_prompts_batch(
-        config=config,
-        provider="openai",
-        input_dir="prompts",  # Directory containing .txt files
-        cache_dir="cache",
-        force=False  # Use cached responses if available
-    )
-    
-    return results
+config = LLMConfig(
+    model_name="gpt-4o-mini",
+    temperature=1.0,
+    max_completion_tokens=200
+)
+
+# Process all .txt files in a directory
+results = process_prompts_batch(
+    config=config,
+    provider="openai",
+    input_dir="prompts",  # Directory containing .txt files
+    cache_dir="cache",
+    force=False  # Use cached responses if available
+)
 
-asyncio.run(process_files())
+print(f"Processed {len(results)} prompts from files")
 ```
 
 ### Custom Verification
@@ -183,7 +228,7 @@ def verify_response(prompt_id, llm_response_data, original_prompt_text, **kwargs
 
 config = LLMConfig(
     model_name="gpt-4o-mini",
-    temperature=0.7,
+    temperature=1.0,
     verification_callback=verify_response,
     verification_callback_args={"min_length": 20}
 )
@@ -200,7 +245,7 @@ Configuration class for LLM requests.
 ```python
 LLMConfig(
     model_name: str,
-    temperature: float = 0.7,
+    temperature: float = 1.0,
     max_completion_tokens: Optional[int] = None,  # Preferred parameter
     max_tokens: Optional[int] = None,  # Deprecated, kept for backward compatibility
     system_instruction: Optional[str] = None,
@@ -213,12 +258,28 @@ LLMConfig(
 
 ### process_prompts_batch
 
-Main function for batch processing of prompts.
+Main function for batch processing of prompts (async operations handled implicitly).
 
 ```python
-async def process_prompts_batch(
+def process_prompts_batch(
     config: LLMConfig,
-    provider: str,  # "openai" or "together"
+    provider: str,  # "openai", "openrouter" (recommended), or "together"
+    prompts: Optional[List[str]] = None,
+    input_dir: Optional[str] = None,
+    cache_dir: str = "llm_cache",
+    force: bool = False,
+    desc: str = "Processing prompts"
+) -> Dict[str, Dict[str, Any]]
+```
+
+### process_prompts_batch_async
+
+Async version for backward compatibility and advanced use cases.
+
+```python
+async def process_prompts_batch_async(
+    config: LLMConfig,
+    provider: str,  # "openai", "openrouter" (recommended), or "together"
     prompts: Optional[List[str]] = None,
     input_dir: Optional[str] = None,
     cache_dir: str = "llm_cache",
@@ -270,10 +331,15 @@ llm_batch_helper/
 ## Supported Models
 
 ### OpenAI
-- gpt-4o-mini
-- gpt-4o
-- gpt-4
-- gpt-3.5-turbo
+- **All OpenAI models** 
+
+### OpenRouter (Recommended - 100+ Models)
+- **OpenAI models**: `openai/gpt-4o`, `openai/gpt-4o-mini`
+- **Anthropic models**: `anthropic/claude-3-5-sonnet`, `anthropic/claude-3-haiku`
+- **DeepSeek models**: `deepseek/deepseek-v3.1-base`, `deepseek/deepseek-chat`
+- **Meta models**: `meta-llama/llama-3.1-405b-instruct`
+- **Google models**: `google/gemini-pro-1.5`
+- **And 90+ more models** from all major providers
 
 ### Together.ai
 - meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
@@ -290,7 +356,7 @@ llm_batch_helper/
 - [API Reference](https://llm-batch-helper.readthedocs.io/en/latest/api.html) - Complete API documentation  
 - [Examples](https://llm-batch-helper.readthedocs.io/en/latest/examples.html) - Practical usage examples
 - [Tutorials](https://llm-batch-helper.readthedocs.io/en/latest/tutorials.html) - Step-by-step tutorials
-- [Provider Guide](https://llm-batch-helper.readthedocs.io/en/latest/providers.html) - OpenAI & Together.ai setup
+- [Provider Guide](https://llm-batch-helper.readthedocs.io/en/latest/providers.html) - OpenAI, OpenRouter & Together.ai setup
 
 ## Contributing
 
@@ -307,6 +373,19 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
+### v0.3.0
+- **🎉 Major Update**: Simplified API - async operations handled implicitly, no async/await required!
+- **📓 Jupyter Support**: Works seamlessly in notebooks without event loop issues
+- **🔍 Detailed Retry Logging**: See exactly what happens during retries with timestamps
+- **🔄 Backward Compatibility**: Original async API still available as `process_prompts_batch_async`
+- **📚 Updated Examples**: All documentation updated to show simplified usage
+- **⚡ Smart Event Loop Handling**: Automatically detects and handles different Python environments
+
+### v0.2.0
+- Enhanced API stability
+- Improved error handling
+- Better documentation
+
 ### v0.1.5
 - Added Together.ai provider support
 - Support for open-source models (Llama, Mixtral, etc.)

{llm_batch_helper-0.1.6 → llm_batch_helper-0.3.0}/llm_batch_helper/__init__.py

@@ -1,15 +1,16 @@
 from .cache import LLMCache
 from .config import LLMConfig
 from .input_handlers import get_prompts, read_prompt_files, read_prompt_list
-from .providers import process_prompts_batch
+from .providers import process_prompts_batch, process_prompts_batch_async
 
-__version__ = "0.1.6"
+__version__ = "0.3.0"
 
 __all__ = [
     "LLMCache",
     "LLMConfig",
     "get_prompts",
     "process_prompts_batch",
+    "process_prompts_batch_async",  # For backward compatibility
     "read_prompt_files",
     "read_prompt_list",
 ]

{llm_batch_helper-0.1.6 → llm_batch_helper-0.3.0}/llm_batch_helper/config.py

@@ -8,7 +8,7 @@ class LLMConfig:
     def __init__(
         self,
         model_name: str,
-        temperature: float = 0.7,
+        temperature: float = 1.0,
         max_tokens: Optional[int] = None,
         system_instruction: Optional[str] = None,
         max_retries: int = 10,  # Max retries for the combined LLM call + Verification
@@ -16,6 +16,7 @@ class LLMConfig:
         verification_callback: Optional[Callable[..., bool]] = None,
         verification_callback_args: Optional[Dict] = None,
         max_completion_tokens: Optional[int] = None,
+        **kwargs
     ):
         self.model_name = model_name
         self.temperature = temperature
@@ -30,3 +31,4 @@ class LLMConfig:
         self.verification_callback_args = (
             verification_callback_args if verification_callback_args is not None else {}
         )
+        self.kwargs = kwargs

{llm_batch_helper-0.1.6 → llm_batch_helper-0.3.0}/llm_batch_helper/providers.py

@@ -1,10 +1,12 @@
 import asyncio
 import os
 from typing import Any, Dict, List, Optional, Tuple, Union
+from datetime import datetime
+import warnings
 
 import httpx
 import openai
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential, before_sleep_log
 from tqdm.asyncio import tqdm_asyncio
 
 from .cache import LLMCache
@@ -12,6 +14,55 @@ from .config import LLMConfig
 from .input_handlers import get_prompts
 
 
+def _run_async_function(async_func, *args, **kwargs):
+    """
+    Run an async function in a way that works in both regular Python and Jupyter notebooks.
+    
+    This handles the event loop management properly for different environments.
+    """
+    try:
+        # Try to get the current event loop
+        loop = asyncio.get_running_loop()
+        # If we're in a running loop (like Jupyter), we need to use nest_asyncio
+        try:
+            import nest_asyncio
+            nest_asyncio.apply()
+            return asyncio.run(async_func(*args, **kwargs))
+        except ImportError:
+            # If nest_asyncio is not available, try to run in the current loop
+            # This is a fallback that might work in some cases
+            import concurrent.futures
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                future = executor.submit(asyncio.run, async_func(*args, **kwargs))
+                return future.result()
+    except RuntimeError:
+        # No event loop running, we can use asyncio.run directly
+        return asyncio.run(async_func(*args, **kwargs))
+
+
+def log_retry_attempt(retry_state):
+    """Custom logging function for retry attempts."""
+    attempt_number = retry_state.attempt_number
+    exception = retry_state.outcome.exception()
+    wait_time = retry_state.next_action.sleep if retry_state.next_action else 0
+    
+    error_type = type(exception).__name__
+    error_msg = str(exception)
+    
+    # Extract status code if available
+    status_code = "unknown"
+    if hasattr(exception, 'status_code'):
+        status_code = exception.status_code
+    elif hasattr(exception, 'response') and hasattr(exception.response, 'status_code'):
+        status_code = exception.response.status_code
+    
+    print(f"🔄 [{datetime.now().strftime('%H:%M:%S')}] Retry attempt {attempt_number}/5:")
+    print(f"   Error: {error_type} (status: {status_code})")
+    print(f"   Message: {error_msg[:100]}{'...' if len(error_msg) > 100 else ''}")
+    print(f"   Waiting {wait_time:.1f}s before next attempt...")
+    print()
+
+
 @retry(
     stop=stop_after_attempt(5),
     wait=wait_exponential(multiplier=1, min=4, max=60),
@@ -25,6 +76,7 @@ from .input_handlers import get_prompts
             openai.APIError,
         )
     ),
+    before_sleep=log_retry_attempt,
     reraise=True,
 )
 async def _get_openai_response_direct(
@@ -46,6 +98,7 @@ async def _get_openai_response_direct(
             messages=messages,
             temperature=config.temperature,
             max_completion_tokens=config.max_completion_tokens,
+            **config.kwargs,
         )
         usage_details = {
             "prompt_token_count": response.usage.prompt_tokens,
@@ -94,6 +147,7 @@ async def _get_together_response_direct(
             "messages": messages,
             "temperature": config.temperature,
             "max_tokens": config.max_completion_tokens,
+            **config.kwargs,
         }
 
         response = await client.post(
@@ -116,6 +170,67 @@ async def _get_together_response_direct(
             "usage_details": usage_details,
         }
 
+
+@retry(
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(multiplier=1, min=4, max=60),
+    retry=retry_if_exception_type(
+        (
+            ConnectionError,
+            TimeoutError,
+            httpx.HTTPStatusError,
+            httpx.RequestError,
+        )
+    ),
+    before_sleep=log_retry_attempt,
+    reraise=True,
+)
+async def _get_openrouter_response_direct(
+    prompt: str, config: LLMConfig
+) -> Dict[str, Union[str, Dict]]:
+    api_key = os.environ.get("OPENROUTER_API_KEY")
+    if not api_key:
+        raise ValueError("OPENROUTER_API_KEY environment variable not set")
+
+    async with httpx.AsyncClient(timeout=1000.0) as client:
+        messages = [
+            {"role": "system", "content": config.system_instruction},
+            {"role": "user", "content": prompt},
+        ]
+
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+
+        payload = {
+            "model": config.model_name,
+            "messages": messages,
+            "temperature": config.temperature,
+            "max_tokens": config.max_completion_tokens,
+            **config.kwargs,
+        }
+
+        response = await client.post(
+            "https://openrouter.ai/api/v1/chat/completions",
+            json=payload,
+            headers=headers,
+        )
+        response.raise_for_status()
+        
+        response_data = response.json()
+        usage = response_data.get("usage", {})
+        usage_details = {
+            "prompt_token_count": usage.get("prompt_tokens", 0),
+            "completion_token_count": usage.get("completion_tokens", 0),
+            "total_token_count": usage.get("total_tokens", 0),
+        }
+        
+        return {
+            "response_text": response_data["choices"][0]["message"]["content"],
+            "usage_details": usage_details,
+        }
+
 async def get_llm_response_with_internal_retry(
     prompt_id: str,
     prompt: str,
@@ -135,6 +250,8 @@ async def get_llm_response_with_internal_retry(
             response = await _get_openai_response_direct(prompt, config)
         elif provider.lower() == "together":
             response = await _get_together_response_direct(prompt, config)
+        elif provider.lower() == "openrouter":
+            response = await _get_openrouter_response_direct(prompt, config)
         else:
             raise ValueError(f"Unsupported provider: {provider}")
 
@@ -150,7 +267,7 @@ async def get_llm_response_with_internal_retry(
         }
 
 
-async def process_prompts_batch(
+async def process_prompts_batch_async(
     prompts: Optional[List[Union[str, Tuple[str, str], Dict[str, Any]]]] = None,
     input_dir: Optional[str] = None,
     config: LLMConfig = None,
@@ -165,7 +282,7 @@ async def process_prompts_batch(
         prompts: Optional list of prompts in any supported format (string, tuple, or dict)
         input_dir: Optional path to directory containing prompt files
         config: LLM configuration
-        provider: LLM provider to use ("openai", "together", or "gemini")
+        provider: LLM provider to use ("openai", "together", or "openrouter")
         desc: Description for progress bar
         cache_dir: Optional directory for caching responses
         force: If True, force regeneration even if cached response exists
@@ -206,6 +323,57 @@ async def process_prompts_batch(
     return results
 
 
+def process_prompts_batch(
+    prompts: Optional[List[Union[str, Tuple[str, str], Dict[str, Any]]]] = None,
+    input_dir: Optional[str] = None,
+    config: LLMConfig = None,
+    provider: str = "openai",
+    desc: str = "Processing prompts",
+    cache_dir: Optional[str] = None,
+    force: bool = False,
+) -> Dict[str, Dict[str, Union[str, Dict]]]:
+    """
+    Process a batch of prompts through the LLM (synchronous version).
+    
+    This is the main user-facing function that works in both regular Python scripts
+    and Jupyter notebooks without requiring async/await syntax.
+    
+    Args:
+        prompts: Optional list of prompts in any supported format (string, tuple, or dict)
+        input_dir: Optional path to directory containing prompt files
+        config: LLM configuration
+        provider: LLM provider to use ("openai", "together", or "openrouter")
+        desc: Description for progress bar
+        cache_dir: Optional directory for caching responses
+        force: If True, force regeneration even if cached response exists
+
+    Returns:
+        Dict mapping prompt IDs to their responses
+
+    Note:
+        Either prompts or input_dir must be provided, but not both.
+        
+    Example:
+        >>> from llm_batch_helper import LLMConfig, process_prompts_batch
+        >>> config = LLMConfig(model_name="gpt-4o-mini")
+        >>> results = process_prompts_batch(
+        ...     prompts=["What is 2+2?", "What is the capital of France?"],
+        ...     config=config,
+        ...     provider="openai"
+        ... )
+    """
+    return _run_async_function(
+        process_prompts_batch_async,
+        prompts=prompts,
+        input_dir=input_dir,
+        config=config,
+        provider=provider,
+        desc=desc,
+        cache_dir=cache_dir,
+        force=force,
+    )
+
+
 async def _process_single_prompt_attempt_with_verification(
     prompt_id: str,
     prompt_text: str,
@@ -238,6 +406,9 @@ async def _process_single_prompt_attempt_with_verification(
         # Process the prompt
         last_exception_details = None
         for attempt in range(config.max_retries):
+            if attempt > 0:
+                print(f"🔁 [{datetime.now().strftime('%H:%M:%S')}] Application-level retry {attempt+1}/{config.max_retries} for prompt: {prompt_id}")
+            
             try:
                 # Get LLM response
                 llm_response_data = await get_llm_response_with_internal_retry(
@@ -245,7 +416,12 @@ async def _process_single_prompt_attempt_with_verification(
                 )
 
                 if "error" in llm_response_data:
+                    print(f"❌ [{datetime.now().strftime('%H:%M:%S')}] API call failed on attempt {attempt+1}: {llm_response_data.get('error', 'Unknown error')}")
                     last_exception_details = llm_response_data
+                    if attempt < config.max_retries - 1:
+                        wait_time = min(2 * 2**attempt, 30)
+                        print(f"   Waiting {wait_time}s before next application retry...")
+                        await asyncio.sleep(wait_time)
                     continue
 
                 # Verify response if callback provided
@@ -265,7 +441,6 @@ async def _process_single_prompt_attempt_with_verification(
                         }
                         if attempt == config.max_retries - 1:
                             return prompt_id, last_exception_details
-                        await asyncio.sleep(min(2 * 2**attempt, 30))
                         continue
 
                 # Save to cache if cache_dir provided
@@ -282,7 +457,7 @@ async def _process_single_prompt_attempt_with_verification(
                 }
                 if attempt == config.max_retries - 1:
                     return prompt_id, last_exception_details
-                await asyncio.sleep(min(2 * 2**attempt, 30))
+                # Sleep is now handled above with logging
                 continue
 
         return prompt_id, last_exception_details or {

{llm_batch_helper-0.1.6 → llm_batch_helper-0.3.0}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "llm_batch_helper"
-version = "0.1.6"
-description = "A Python package that enables batch submission of prompts to LLM APIs, with built-in async capabilities and response caching."
+version = "0.3.0"
+description = "A Python package that enables batch submission of prompts to LLM APIs, with simplified interface and built-in async capabilities handled implicitly."
 authors = ["Tianyi Peng <tianyipeng95@gmail.com>"]
 readme = "README.md"
 license = "MIT"
 homepage = "https://github.com/TianyiPeng/LLM_batch_helper"
 repository = "https://github.com/TianyiPeng/LLM_batch_helper"
-keywords = ["llm", "openai", "together", "batch", "async", "ai", "nlp", "api"]
+keywords = ["llm", "openai", "together", "openrouter", "batch", "async", "ai", "nlp", "api"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",