batchwizard 0.1.0__tar.gz

batchwizard-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Carl Kugblenu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

batchwizard-0.1.0/PKG-INFO

@@ -0,0 +1,172 @@
+Metadata-Version: 2.1
+Name: batchwizard
+Version: 0.1.0
+Summary: BatchWizard: Manage OpenAI batch processing jobs with ease
+Home-page: https://github.com/cmakafui/batchwizard
+License: MIT
+Keywords: openai,batch,cli,async
+Author: Carl Kugblenu
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
+Requires-Dist: loguru (>=0.7.2,<0.8.0)
+Requires-Dist: openai (>=1.37.0,<2.0.0)
+Requires-Dist: pydantic (>=2.8.2,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.3.4,<3.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: rich (>=13.7.1,<14.0.0)
+Requires-Dist: typer (>=0.12.3,<0.13.0)
+Project-URL: Repository, https://github.com/cmakafui/batchwizard
+Description-Content-Type: text/markdown
+
+# BatchWizard
+
+BatchWizard is a powerful CLI tool for managing OpenAI batch processing jobs with ease. It provides functionalities to upload files, create batch jobs, check their status, and download the results. The tool uses asynchronous processing to efficiently handle multiple jobs concurrently.
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Usage](#usage)
+- [Configuration](#configuration)
+- [Commands](#commands)
+- [Features](#features)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Installation
+
+You can install BatchWizard using `pipx` for an isolated environment or directly via `pip`.
+
+### Using pipx (recommended)
+
+```bash
+pipx install batchwizard
+```
+
+### Using pip
+
+```bash
+pip install batchwizard
+```
+
+Ensure you have `pipx` or `pip` installed on your system. For `pipx`, you can follow the installation instructions [here](https://pipx.pypa.io/stable/installation/).
+
+## Usage
+
+BatchWizard provides a command-line interface (CLI) for managing batch jobs. Here are some example commands:
+
+### Process Batch Jobs
+
+To process a directory containing input files:
+
+```bash
+batchwizard process <input_directory> [--output-directory OUTPUT_DIR] [--max-concurrent-jobs NUM] [--check-interval SECONDS]
+```
+
+### List Recent Jobs
+
+To list recent batch jobs:
+
+```bash
+batchwizard list-jobs [--limit NUM] [--all]
+```
+
+### Cancel a Job
+
+To cancel a specific batch job:
+
+```bash
+batchwizard cancel <job_id>
+```
+
+### Download Job Results
+
+To download results for a completed batch job:
+
+```bash
+batchwizard download <job_id> [--output-file FILE_PATH]
+```
+
+## Configuration
+
+### Setting up the OpenAI API Key
+
+To set the OpenAI API key:
+
+```bash
+batchwizard configure --set-key YOUR_API_KEY
+```
+
+### Show Current Configuration
+
+To show the current configuration:
+
+```bash
+batchwizard configure --show
+```
+
+### Reset Configuration
+
+To reset the configuration to default values:
+
+```bash
+batchwizard configure --reset
+```
+
+## Commands
+
+BatchWizard supports the following commands:
+
+- `process`: Process batch jobs from input files in the specified directory.
+- `configure`: Manage BatchWizard configuration.
+- `list-jobs`: List recent batch jobs.
+- `cancel`: Cancel a specific batch job.
+- `download`: Download results for a completed batch job.
+
+For detailed information on each command, use the `--help` option:
+
+```bash
+batchwizard <command> --help
+```
+
+## Features
+
+- **Asynchronous Processing**: Efficiently handle multiple batch jobs concurrently.
+- **Rich UI**: Display progress and job status using a rich, interactive interface.
+- **Flexible Configuration**: Easily manage API keys and other settings.
+- **Job Management**: List, cancel, and download results for batch jobs.
+- **Error Handling**: Robust error handling and informative error messages.
+
+## Contributing
+
+We welcome contributions to BatchWizard! To contribute, follow these steps:
+
+1. Fork the repository.
+2. Create a new branch: `git checkout -b feature/your-feature-name`.
+3. Make your changes and commit them: `git commit -m 'Add some feature'`.
+4. Push to the branch: `git push origin feature/your-feature-name`.
+5. Open a pull request.
+
+### Running Tests
+
+To run tests, use `pytest`:
+
+```bash
+pytest --cov=batchwizard tests/
+```
+
+Ensure your code passes all tests and meets the coding standards before opening a pull request.
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+## Contact
+
+For any questions or feedback, feel free to open an issue on the [GitHub repository](https://github.com/cmakafui/batchwizard).
+

batchwizard-0.1.0/README.md

@@ -0,0 +1,145 @@
+# BatchWizard
+
+BatchWizard is a powerful CLI tool for managing OpenAI batch processing jobs with ease. It provides functionalities to upload files, create batch jobs, check their status, and download the results. The tool uses asynchronous processing to efficiently handle multiple jobs concurrently.
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Usage](#usage)
+- [Configuration](#configuration)
+- [Commands](#commands)
+- [Features](#features)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Installation
+
+You can install BatchWizard using `pipx` for an isolated environment or directly via `pip`.
+
+### Using pipx (recommended)
+
+```bash
+pipx install batchwizard
+```
+
+### Using pip
+
+```bash
+pip install batchwizard
+```
+
+Ensure you have `pipx` or `pip` installed on your system. For `pipx`, you can follow the installation instructions [here](https://pipx.pypa.io/stable/installation/).
+
+## Usage
+
+BatchWizard provides a command-line interface (CLI) for managing batch jobs. Here are some example commands:
+
+### Process Batch Jobs
+
+To process a directory containing input files:
+
+```bash
+batchwizard process <input_directory> [--output-directory OUTPUT_DIR] [--max-concurrent-jobs NUM] [--check-interval SECONDS]
+```
+
+### List Recent Jobs
+
+To list recent batch jobs:
+
+```bash
+batchwizard list-jobs [--limit NUM] [--all]
+```
+
+### Cancel a Job
+
+To cancel a specific batch job:
+
+```bash
+batchwizard cancel <job_id>
+```
+
+### Download Job Results
+
+To download results for a completed batch job:
+
+```bash
+batchwizard download <job_id> [--output-file FILE_PATH]
+```
+
+## Configuration
+
+### Setting up the OpenAI API Key
+
+To set the OpenAI API key:
+
+```bash
+batchwizard configure --set-key YOUR_API_KEY
+```
+
+### Show Current Configuration
+
+To show the current configuration:
+
+```bash
+batchwizard configure --show
+```
+
+### Reset Configuration
+
+To reset the configuration to default values:
+
+```bash
+batchwizard configure --reset
+```
+
+## Commands
+
+BatchWizard supports the following commands:
+
+- `process`: Process batch jobs from input files in the specified directory.
+- `configure`: Manage BatchWizard configuration.
+- `list-jobs`: List recent batch jobs.
+- `cancel`: Cancel a specific batch job.
+- `download`: Download results for a completed batch job.
+
+For detailed information on each command, use the `--help` option:
+
+```bash
+batchwizard <command> --help
+```
+
+## Features
+
+- **Asynchronous Processing**: Efficiently handle multiple batch jobs concurrently.
+- **Rich UI**: Display progress and job status using a rich, interactive interface.
+- **Flexible Configuration**: Easily manage API keys and other settings.
+- **Job Management**: List, cancel, and download results for batch jobs.
+- **Error Handling**: Robust error handling and informative error messages.
+
+## Contributing
+
+We welcome contributions to BatchWizard! To contribute, follow these steps:
+
+1. Fork the repository.
+2. Create a new branch: `git checkout -b feature/your-feature-name`.
+3. Make your changes and commit them: `git commit -m 'Add some feature'`.
+4. Push to the branch: `git push origin feature/your-feature-name`.
+5. Open a pull request.
+
+### Running Tests
+
+To run tests, use `pytest`:
+
+```bash
+pytest --cov=batchwizard tests/
+```
+
+Ensure your code passes all tests and meets the coding standards before opening a pull request.
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+## Contact
+
+For any questions or feedback, feel free to open an issue on the [GitHub repository](https://github.com/cmakafui/batchwizard).

batchwizard-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[tool.poetry]
+name = "batchwizard"
+version = "0.1.0"
+description = "BatchWizard: Manage OpenAI batch processing jobs with ease"
+authors = ["Carl Kugblenu"]
+license = "MIT"
+readme = "README.md"
+homepage = "https://github.com/cmakafui/batchwizard"
+repository = "https://github.com/cmakafui/batchwizard"
+keywords = ["openai", "batch", "cli", "async"]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+typer = "^0.12.3"
+rich = "^13.7.1"
+openai = "^1.37.0"
+pydantic = "^2.8.2"
+aiofiles = "^24.1.0"
+loguru = "^0.7.2"
+pydantic-settings = "^2.3.4"
+python-dotenv = "^1.0.1"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.2"
+pytest-asyncio = "^0.23.8"
+isort = "^5.13.2"
+black = "^24.4.2"
+ruff = "^0.5.5"
+pytest-mock = "^3.14.0"
+
+[tool.poetry.scripts]
+batchwizard = "batchwizard.cli:app"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

batchwizard-0.1.0/src/batchwizard/__init__.py

@@ -0,0 +1,6 @@
+# __init__.py
+from .cli import app
+from .models import BatchJob, BatchJobResult
+from .processor import BatchProcessor
+
+__all__ = ["BatchProcessor", "BatchJob", "BatchJobResult", "app"]

batchwizard-0.1.0/src/batchwizard/cli.py

@@ -0,0 +1,191 @@
+# cli.py
+import asyncio
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from .config import BatchWizardSettings, config
+from .processor import BatchProcessor
+from .ui import BatchWizardUI
+from .utils import get_api_key, set_api_key, setup_logger
+
+app = typer.Typer(help="BatchWizard: Manage OpenAI batch processing jobs with ease")
+console = Console()
+logger = setup_logger(console)
+
+
+@app.command()
+def process(
+    input_directory: Path = typer.Argument(
+        ..., help="Directory containing input files for processing"
+    ),
+    output_directory: Optional[Path] = typer.Option(
+        None, help="Directory to store output files"
+    ),
+    max_concurrent_jobs: int = typer.Option(
+        5, help="Maximum number of concurrent jobs"
+    ),
+    check_interval: int = typer.Option(
+        5, help="Initial interval (in seconds) between job status checks"
+    ),
+):
+    """Process batch jobs from input files in the specified directory."""
+    if not output_directory:
+        output_directory = input_directory / "results"
+    output_directory.mkdir(parents=True, exist_ok=True)
+
+    api_key = get_api_key()
+    if not api_key:
+        logger.error(
+            "API key not set. Please set it using 'batchwizard configure --set-key YOUR_API_KEY'"
+        )
+        raise typer.Exit(code=1)
+
+    config.settings.max_concurrent_jobs = max_concurrent_jobs
+    config.settings.check_interval = check_interval
+    config.save()
+
+    processor = BatchProcessor()
+    ui = BatchWizardUI(Console())
+
+    async def run_and_close():
+        try:
+            await ui.run_processing(processor, input_directory, output_directory)
+        finally:
+            await processor.close()
+
+    asyncio.run(run_and_close())
+
+
+@app.command()
+def configure(
+    set_key: Optional[str] = typer.Option(
+        None, "--set-key", help="Set the OpenAI API key"
+    ),
+    show: bool = typer.Option(False, "--show", help="Show the current configuration"),
+    reset: bool = typer.Option(
+        False, "--reset", help="Reset the configuration to default values"
+    ),
+):
+    """Manage BatchWizard configuration."""
+    if set_key:
+        set_api_key(set_key)
+        console.print("[green]API key set successfully.[/green]")
+    elif show:
+        api_key = get_api_key()
+        masked_key = f"{api_key[:4]}...{api_key[-4:]}" if api_key else "Not set"
+        console.print(f"API Key: {masked_key}")
+        console.print(f"Max Concurrent Jobs: {config.settings.max_concurrent_jobs}")
+        console.print(f"Check Interval: {config.settings.check_interval} seconds")
+    elif reset:
+        config.settings = BatchWizardSettings()
+        config.save()
+        console.print("[yellow]Configuration reset to default values.[/yellow]")
+    else:
+        console.print(
+            "Use --set-key, --show, or --reset options to manage configuration."
+        )
+
+
+@app.command()
+def list_jobs(
+    limit: int = typer.Option(100, help="Number of jobs to display"),
+    all: bool = typer.Option(False, "--all", help="Display all jobs"),
+):
+    """List recent batch jobs."""
+
+    async def fetch_jobs():
+        processor = BatchProcessor()
+        console = Console()  # Create a Console object directly
+        try:
+            jobs = await processor.client.batches.list(limit=None if all else limit)
+            table = Table(title="Batch Jobs")
+            table.add_column("Job ID", style="cyan")
+            table.add_column("Status", style="magenta")
+            table.add_column("Created At", style="green")
+            table.add_column("Completed", style="blue")
+            table.add_column("Failed", style="red")
+
+            for job in jobs.data:
+                created_at = datetime.fromtimestamp(job.created_at).strftime(
+                    "%Y-%m-%d %H:%M:%S"
+                )
+                table.add_row(
+                    job.id,
+                    job.status,
+                    created_at,
+                    str(job.request_counts.completed),
+                    str(job.request_counts.failed),
+                )
+            console.print(table)  # Use the console object to print the table
+        finally:
+            await processor.close()
+
+    asyncio.run(fetch_jobs())
+
+
+@app.command()
+def cancel(
+    job_id: str = typer.Argument(..., help="ID of the batch job to cancel"),
+):
+    """Cancel a specific batch job."""
+
+    async def cancel_job():
+        processor = BatchProcessor()
+        try:
+            await processor.client.batches.cancel(job_id)
+            console.print(f"[green]Job {job_id} cancelled successfully.[/green]")
+        except Exception as e:
+            console.print(f"[red]Error cancelling job {job_id}: {str(e)}[/red]")
+        finally:
+            await processor.close()
+
+    asyncio.run(cancel_job())
+
+
+@app.command()
+def download(
+    job_id: str = typer.Argument(
+        ..., help="ID of the batch job to download results for"
+    ),
+    output_file: Path = typer.Option(
+        None, help="Path to save the output file (default: <job_id>_results.jsonl)"
+    ),
+):
+    """Download results for a completed batch job."""
+    if not output_file:
+        output_file = Path(f"{job_id}_results.jsonl")
+
+    async def download_results():
+        processor = BatchProcessor()
+        try:
+            batch_job = await processor.client.batches.retrieve(job_id)
+            if batch_job.status != "completed":
+                console.print(
+                    f"[yellow]Job {job_id} is not completed (status: {batch_job.status}). Cannot download results.[/yellow]"
+                )
+                return
+
+            success = await processor.download_batch_results(batch_job, output_file)
+            if success:
+                console.print(
+                    f"[green]Results for job {job_id} downloaded successfully to {output_file}[/green]"
+                )
+            else:
+                console.print(f"[red]Failed to download results for job {job_id}[/red]")
+        except Exception as e:
+            console.print(
+                f"[red]Error downloading results for job {job_id}: {str(e)}[/red]"
+            )
+        finally:
+            await processor.close()
+
+    asyncio.run(download_results())
+
+
+if __name__ == "__main__":
+    app()

batchwizard-0.1.0/src/batchwizard/config.py

@@ -0,0 +1,48 @@
+# config.py
+import os
+from pathlib import Path
+from typing import Optional
+
+import typer
+from pydantic import BaseModel, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class BatchWizardSettings(BaseSettings):
+    api_key: Optional[str] = Field(None, env="OPENAI_API_KEY")
+    max_concurrent_jobs: int = 5
+    check_interval: int = 5
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
+
+
+class Config(BaseModel):
+    settings: BatchWizardSettings = BatchWizardSettings()
+
+    @property
+    def config_dir(self) -> Path:
+        return Path(typer.get_app_dir("BatchWizard"))
+
+    @property
+    def config_file(self) -> Path:
+        return self.config_dir / "config.json"
+
+    def load(self) -> None:
+        if self.config_file.exists():
+            self.settings = BatchWizardSettings.model_validate_json(
+                self.config_file.read_text()
+            )
+
+    def save(self) -> None:
+        self.config_dir.mkdir(parents=True, exist_ok=True)
+        self.config_file.write_text(self.settings.model_dump_json())
+
+    def get_api_key(self) -> Optional[str]:
+        return self.settings.api_key or os.getenv("OPENAI_API_KEY")
+
+    def set_api_key(self, api_key: str) -> None:
+        self.settings.api_key = api_key
+        self.save()
+
+
+config = Config()
+config.load()

batchwizard-0.1.0/src/batchwizard/models.py

@@ -0,0 +1,18 @@
+# models.py
+from pathlib import Path
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class BatchJob(BaseModel):
+    id: str
+    status: str
+    input_file_id: str
+    output_file_id: Optional[str] = None
+
+
+class BatchJobResult(BaseModel):
+    job_id: str
+    success: bool
+    output_file_path: Optional[Path] = None

batchwizard-0.1.0/src/batchwizard/processor.py

@@ -0,0 +1,145 @@
+# processor.py
+import asyncio
+from pathlib import Path
+from typing import List, Optional
+
+import aiofiles
+from loguru import logger
+from openai import AsyncOpenAI
+
+from .config import config
+from .models import BatchJob, BatchJobResult
+
+
+class BatchProcessor:
+    def __init__(self):
+        self.client = AsyncOpenAI(api_key=config.get_api_key())
+        self.settings = config.settings
+
+    async def upload_file(self, file_path: Path) -> Optional[str]:
+        try:
+            async with aiofiles.open(file_path, "rb") as file:
+                file_content = await file.read()
+
+            response = await self.client.files.create(
+                file=(file_path.name, file_content), purpose="batch"
+            )
+            logger.info(
+                f"File uploaded successfully: {response.id}, Filename: {file_path.name}"
+            )
+            return response.id
+        except Exception as e:
+            logger.error(f"Error uploading file {file_path.name}: {str(e)}")
+            return None
+
+    async def create_batch_job(self, input_file_id: str) -> Optional[BatchJob]:
+        try:
+            batch_job = await self.client.batches.create(
+                input_file_id=input_file_id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+            logger.info(f"Created batch job with ID: {batch_job.id}")
+            return BatchJob(
+                id=batch_job.id,
+                status=self.normalize_status(batch_job.status),
+                input_file_id=input_file_id,
+            )
+        except Exception as e:
+            logger.error(f"Error creating batch job: {str(e)}")
+            return None
+
+    async def check_batch_status(self, batch_id: str) -> Optional[str]:
+        try:
+            batch_job = await self.client.batches.retrieve(batch_id)
+            return self.normalize_status(batch_job.status)
+        except Exception as e:
+            logger.error(f"Error checking batch status: {str(e)}")
+            return None
+
+    def normalize_status(self, status: str) -> str:
+        """Normalize the status string to lowercase with underscores."""
+        return status.lower().replace(" ", "_")
+
+    async def download_batch_results(self, batch_job, output_file_path: Path) -> bool:
+        try:
+            if batch_job.status == "completed" and batch_job.output_file_id:
+                result = await self.client.files.content(batch_job.output_file_id)
+                async with aiofiles.open(output_file_path, "wb") as file:
+                    await file.write(result.content)
+                logger.info(f"Downloaded results to {output_file_path}")
+                return True
+            else:
+                logger.warning(
+                    f"Batch job not completed or missing output file. Status: {batch_job.status}"
+                )
+                return False
+        except Exception as e:
+            logger.error(f"Error downloading batch results: {str(e)}")
+            return False
+
+    async def process_batch_job(
+        self, batch_job: BatchJob, output_dir: Path
+    ) -> BatchJobResult:
+        check_interval = self.settings.check_interval
+        while True:
+            status = await self.check_batch_status(batch_job.id)
+            if status == "completed":
+                try:
+                    batch_job = await self.client.batches.retrieve(batch_job.id)
+                    if batch_job.output_file_id:
+                        output_file = output_dir / f"{batch_job.id}_results.jsonl"
+                        if await self.download_batch_results(batch_job, output_file):
+                            logger.info(
+                                f"Successfully processed batch job {batch_job.id}"
+                            )
+                            return BatchJobResult(
+                                job_id=batch_job.id,
+                                success=True,
+                                output_file_path=output_file,
+                            )
+                    else:
+                        logger.error(
+                            f"No output file ID found for completed batch job {batch_job.id}"
+                        )
+                except Exception as e:
+                    logger.error(
+                        f"Error processing completed batch job {batch_job.id}: {str(e)}"
+                    )
+            elif status in ["failed", "expired", "cancelled"]:
+                logger.error(f"Batch job {batch_job.id} {status}")
+                return BatchJobResult(job_id=batch_job.id, success=False)
+            elif status is None:
+                logger.error(f"Failed to retrieve status for batch job {batch_job.id}")
+                return BatchJobResult(job_id=batch_job.id, success=False)
+
+            await asyncio.sleep(check_interval)
+            check_interval = min(
+                check_interval * 1.5, 60
+            )  # Implement exponential backoff
+
+    async def process_directory(
+        self, input_dir: Path, output_dir: Path
+    ) -> List[BatchJobResult]:
+        input_files = list(input_dir.glob("*_batch_*.jsonl"))
+        if not input_files:
+            logger.warning(f"No input files found in {input_dir}")
+            return []
+
+        semaphore = asyncio.Semaphore(self.settings.max_concurrent_jobs)
+
+        async def process_file(input_file: Path) -> Optional[BatchJobResult]:
+            async with semaphore:
+                file_id = await self.upload_file(input_file)
+                if file_id:
+                    batch_job = await self.create_batch_job(file_id)
+                    if batch_job:
+                        return await self.process_batch_job(batch_job, output_dir)
+            return None
+
+        tasks = [process_file(file) for file in input_files]
+        results = await asyncio.gather(*tasks)
+        return [result for result in results if result is not None]
+
+    async def close(self):
+        await self.client.close()

batchwizard-0.1.0/src/batchwizard/ui.py

@@ -0,0 +1,250 @@
+# ui.py
+import asyncio
+from datetime import datetime
+from pathlib import Path
+from typing import List
+
+from loguru import logger
+from rich.console import Console
+from rich.layout import Layout
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import (BarColumn, Progress, SpinnerColumn,
+                           TaskProgressColumn, TextColumn, TimeElapsedColumn)
+from rich.table import Table
+from rich.text import Text
+
+from .processor import BatchProcessor
+
+
+class BatchWizardUI:
+    def __init__(self, console: Console):
+        self.console = console
+        self.job_table = self.create_job_table()
+        self.stats_table = self.create_stats_table()
+        self.log_messages = []
+        self.jobs = {}  # Dictionary to store job data
+        self.total_jobs = 0
+        self.completed_jobs = 0
+        self.failed_jobs = 0
+
+    def create_layout(self) -> Layout:
+        layout = Layout()
+        layout.split_column(Layout(name="header", size=3), Layout(name="body"))
+        layout["body"].split_row(
+            Layout(name="main", ratio=2), Layout(name="sidebar", ratio=1)
+        )
+        layout["body"]["main"].split_column(
+            Layout(name="progress", ratio=1), Layout(name="job_status", ratio=2)
+        )
+        layout["body"]["sidebar"].split_column(
+            Layout(name="stats", ratio=1), Layout(name="logs", ratio=2)
+        )
+        return layout
+
+    def create_progress_bars(self):
+        overall_progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+        )
+        job_progress = Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+        )
+        return overall_progress, job_progress
+
+    def create_job_table(self):
+        job_table = Table(show_header=True, header_style="bold magenta", expand=True)
+        job_table.add_column("Job ID", style="dim", no_wrap=True)
+        job_table.add_column("Status", style="bold")
+        job_table.add_column("Progress", justify="right")
+        return job_table
+
+    def update_job_status(self, job_id: str, status: str, progress: str):
+        color = (
+            "green"
+            if status == "completed"
+            else "red" if status in ["failed", "expired", "cancelled"] else "yellow"
+        )
+        self.jobs[job_id] = (f"[{color}]{status}", progress)
+        self.job_table = self.create_job_table()
+        for job_id, (status, progress) in self.jobs.items():
+            self.job_table.add_row(job_id, status, progress)
+
+    def create_stats_table(self):
+        stats_table = Table(show_header=False, expand=True)
+        stats_table.add_column("Metric", style="cyan")
+        stats_table.add_column("Value", justify="right")
+        stats_table.add_row("Total Jobs", "0")
+        stats_table.add_row("Completed", "0")
+        stats_table.add_row("In Progress", "0")
+        stats_table.add_row("Failed", "0")
+        return stats_table
+
+    def update_stats(self):
+        self.stats_table.rows.clear()
+        self.stats_table.add_row("Total Jobs", str(self.total_jobs))
+        self.stats_table.add_row("Completed", f"[green]{self.completed_jobs}[/green]")
+        self.stats_table.add_row(
+            "In Progress",
+            f"[yellow]{self.total_jobs - self.completed_jobs - self.failed_jobs}[/yellow]",
+        )
+        self.stats_table.add_row("Failed", f"[red]{self.failed_jobs}[/red]")
+
+    def add_log(self, message: str):
+        self.log_messages.append(message)
+        if len(self.log_messages) > 10:  # Keep only the last 10 log messages
+            self.log_messages.pop(0)
+        logger.info(message)
+
+    def get_log_panel(self):
+        return Panel("\n".join(self.log_messages), title="Logs", border_style="green")
+
+    def update_layout(
+        self,
+        layout: Layout,
+        header: str,
+        overall_progress: Progress,
+        job_table: Table,
+        stats_table: Table,
+        log_panel: Panel,
+    ):
+        layout["header"].update(
+            Panel(Text(header, style="bold blue"), border_style="blue")
+        )
+        layout["body"]["main"]["progress"].update(
+            Panel(overall_progress, title="Overall Progress", border_style="green")
+        )
+        layout["body"]["main"]["job_status"].update(
+            Panel(job_table, title="Job Status", border_style="magenta")
+        )
+        layout["body"]["sidebar"]["stats"].update(
+            Panel(stats_table, title="Statistics", border_style="cyan")
+        )
+        layout["body"]["sidebar"]["logs"].update(log_panel)
+
+    async def run_processing(
+        self, processor: BatchProcessor, input_dir: Path, output_dir: Path
+    ):
+        layout = self.create_layout()
+        overall_progress, job_progress = self.create_progress_bars()
+
+        overall_task = overall_progress.add_task("[bold blue]Processing", total=100)
+        upload_task = overall_progress.add_task("[green]Uploading files", visible=False)
+        process_task = overall_progress.add_task("[cyan]Processing jobs", visible=False)
+
+        async def update_ui():
+            self.update_layout(
+                layout,
+                "BatchWizard Processing",
+                overall_progress,
+                self.job_table,
+                self.stats_table,
+                self.get_log_panel(),
+            )
+
+        with Live(layout, console=self.console, screen=True, refresh_per_second=4):
+            await update_ui()
+
+            input_files = list(input_dir.glob("*_batch_*.jsonl"))
+            if not input_files:
+                self.add_log(f"No input files found in {input_dir}")
+                await update_ui()
+                return
+
+            overall_progress.update(upload_task, total=len(input_files), visible=True)
+            overall_progress.update(process_task, total=len(input_files), visible=True)
+
+            self.total_jobs = len(input_files)
+
+            async def process_file(input_file: Path):
+                file_id = await processor.upload_file(input_file)
+                overall_progress.update(upload_task, advance=1)
+                if file_id:
+                    batch_job = await processor.create_batch_job(file_id)
+                    if batch_job:
+                        self.update_job_status(batch_job.id, batch_job.status, "0%")
+                        self.add_log(f"Job created: {batch_job.id}")
+                        await update_ui()
+
+                        result = await processor.process_batch_job(
+                            batch_job, output_dir
+                        )
+                        if result.success:
+                            self.completed_jobs += 1
+                            self.update_job_status(batch_job.id, "completed", "100%")
+                            self.add_log(f"Job completed: {batch_job.id}")
+                            if result.output_file_path:
+                                self.add_log(
+                                    f"Results saved to: {result.output_file_path}"
+                                )
+                        else:
+                            self.failed_jobs += 1
+                            self.update_job_status(batch_job.id, "failed", "100%")
+                            self.add_log(f"Job failed: {batch_job.id}")
+
+                        overall_progress.update(process_task, advance=1)
+                        self.update_stats()
+                        await update_ui()
+                    else:
+                        self.add_log(
+                            f"Failed to create batch job for {input_file.name}"
+                        )
+                else:
+                    self.add_log(f"Failed to upload file {input_file.name}")
+                await update_ui()
+
+            tasks = [process_file(file) for file in input_files]
+            await asyncio.gather(*tasks)
+
+            overall_progress.update(overall_task, completed=100)
+            await update_ui()
+
+        self.console.print("[bold green]Processing completed![/bold green]")
+        self.console.print(f"Total jobs: {self.total_jobs}")
+        self.console.print(f"Completed jobs: {self.completed_jobs}")
+        self.console.print(f"Failed jobs: {self.failed_jobs}")
+        if self.completed_jobs > 0:
+            self.console.print(f"Results saved in: {output_dir}")
+
+    def display_job_list(self, jobs: List[dict]):
+        table = Table(title="Batch Jobs")
+        table.add_column("Job ID", style="cyan")
+        table.add_column("Status", style="magenta")
+        table.add_column("Created At", style="green")
+        table.add_column("Completed", style="blue")
+        table.add_column("Failed", style="red")
+
+        for job in jobs:
+            created_at = datetime.fromtimestamp(job["created_at"]).strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
+            table.add_row(
+                job["id"],
+                job["status"],
+                created_at,
+                str(job["request_counts"]["completed"]),
+                str(job["request_counts"]["failed"]),
+            )
+
+        self.console.print(table)
+
+    def display_cancel_result(self, job_id: str, success: bool):
+        if success:
+            self.console.print(f"[green]Job {job_id} cancelled successfully.[/green]")
+        else:
+            self.console.print(f"[red]Failed to cancel job {job_id}.[/red]")
+
+    def display_download_result(self, job_id: str, output_file: Path, success: bool):
+        if success:
+            self.console.print(
+                f"[green]Results for job {job_id} downloaded successfully to {output_file}[/green]"
+            )
+        else:
+            self.console.print(
+                f"[red]Failed to download results for job {job_id}[/red]"
+            )

batchwizard-0.1.0/src/batchwizard/utils.py

@@ -0,0 +1,36 @@
+# utils.py
+from typing import Optional
+
+from loguru import logger
+from rich.console import Console
+
+from .config import config
+
+
+def setup_logger(console: Console = None):
+    logger.remove()
+    if console is None:
+        console = Console()
+    logger.add(
+        console.file,
+        format="<red>{time:YYYY-MM-DD HH:mm:ss}</red> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+        level="ERROR",
+        backtrace=True,
+        diagnose=True,
+    )
+    return logger
+
+
+def get_api_key() -> Optional[str]:
+    """Get the API key from config or environment variable."""
+    return config.get_api_key()
+
+
+def set_api_key(api_key: str) -> None:
+    """Set the API key in the config."""
+    config.set_api_key(api_key)
+
+
+def get_settings():
+    """Get the current settings."""
+    return config.settings