tinbox 0.1.0__py3-none-any.whl

tinbox/__init__.py

@@ -0,0 +1,9 @@
+"""Tinbox - A CLI Translation Tool using LLMs.
+
+This package provides functionality for translating documents using various LLM models,
+supporting multiple input formats and translation algorithms.
+"""
+
+__version__ = "0.1.0"
+__author__ = "strickvl"
+__license__ = "MIT"

tinbox/cli.py

@@ -0,0 +1,307 @@
+"""Command-line interface for Tinbox."""
+
+import asyncio
+import sys
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.table import Table
+
+from tinbox.core import (
+    FileType,
+    ModelType,
+    TranslationConfig,
+    TranslationResult,
+    translate_document,
+)
+from tinbox.core.cost import estimate_cost
+from tinbox.core.processor import load_document
+from tinbox.core.translation import create_translator
+from tinbox.core.output import (
+    OutputFormat,
+    TranslationMetadata,
+    TranslationOutput,
+    create_handler,
+)
+from tinbox.utils.logging import configure_logging, get_logger
+
+app = typer.Typer(
+    name="tinbox",
+    help="A CLI tool for translating documents using LLMs",
+    add_completion=False,
+)
+console = Console()
+logger = get_logger()
+
+
+def version_callback(value: bool) -> None:
+    """Print version information and exit."""
+    if value:
+        from tinbox import __version__
+
+        console.print(f"Tinbox version: {__version__}")
+        raise typer.Exit(0)
+
+
+@app.callback()
+def main(
+    version: Optional[bool] = typer.Option(
+        None,
+        "--version",
+        "-v",
+        help="Show version information and exit.",
+        callback=version_callback,
+        is_eager=True,
+    ),
+    log_level: str = typer.Option(
+        "INFO",
+        "--log-level",
+        "-l",
+        help="Set the logging level.",
+    ),
+    json_logs: bool = typer.Option(
+        False,
+        "--json",
+        "-j",
+        help="Output logs in JSON format.",
+    ),
+) -> None:
+    """Tinbox - A CLI Translation Tool using LLMs."""
+    configure_logging(level=log_level, json=json_logs)
+
+
+def display_cost_estimate(estimate, model: ModelType) -> None:
+    """Display cost estimate in a rich format."""
+    table = Table(title="Cost Estimate", show_header=False)
+    table.add_column("Metric", style="bold blue")
+    table.add_column("Value")
+
+    table.add_row("Estimated Tokens", f"{estimate.estimated_tokens:,}")
+    if model != ModelType.OLLAMA:
+        table.add_row("Estimated Cost", f"${estimate.estimated_cost:.2f}")
+    table.add_row("Estimated Time", f"{estimate.estimated_time / 60:.1f} minutes")
+    table.add_row("Cost Level", estimate.cost_level.value.title())
+
+    console.print(table)
+
+    if estimate.warnings:
+        console.print("\n[yellow]Warnings:[/yellow]")
+        for warning in estimate.warnings:
+            console.print(f"• {warning}")
+
+
+def parse_model_spec(model_spec: str) -> tuple[ModelType, str]:
+    """Parse a model specification string.
+
+    Args:
+        model_spec: Model specification (e.g., 'openai:gpt-4o', 'anthropic:claude-3-sonnet', 'ollama:mistral-small')
+
+    Returns:
+        Tuple of (ModelType, str) for the provider and model name.
+
+    Raises:
+        ValueError: If the model specification is invalid
+    """
+    if ":" not in model_spec:
+        raise ValueError(
+            "Invalid model specification. Use format 'provider:model' "
+            "(e.g., 'openai:gpt-4o', 'anthropic:claude-3-sonnet', 'ollama:mistral-small')"
+        )
+
+    provider, model = model_spec.split(":", 1)
+    try:
+        return ModelType(provider.lower()), model
+    except ValueError:
+        raise ValueError(
+            f"Unknown model provider: {provider}. "
+            f"Supported providers: {', '.join(m.value for m in ModelType)}"
+        )
+
+
+@app.command()
+def translate(
+    input_file: Path = typer.Argument(
+        ...,
+        help="The input file to translate.",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        resolve_path=True,
+    ),
+    output_file: Optional[Path] = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="The output file. If not specified, prints to stdout.",
+    ),
+    output_format: OutputFormat = typer.Option(
+        OutputFormat.TEXT,
+        "--format",
+        "-F",
+        help="Output format (text, json, or markdown).",
+    ),
+    source_lang: str = typer.Option(
+        None,
+        "--from",
+        "-f",
+        help="Source language code (e.g., 'en', 'zh'). Defaults to auto-detect.",
+    ),
+    target_lang: str = typer.Option(
+        "en",
+        "--to",
+        "-t",
+        help="Target language code. Defaults to 'en' (English).",
+    ),
+    model: str = typer.Option(
+        "anthropic:claude-3-sonnet",
+        "--model",
+        "-m",
+        help="Model to use (e.g., 'openai:gpt-4o', 'anthropic:claude-3-sonnet', 'ollama:mistral-small').",
+    ),
+    algorithm: str = typer.Option(
+        "page",
+        "--algorithm",
+        "-a",
+        help="Translation algorithm to use: 'page' or 'sliding-window'.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Estimate cost and tokens without performing translation.",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Skip warnings and proceed with translation.",
+    ),
+    max_cost: Optional[float] = typer.Option(
+        None,
+        "--max-cost",
+        help="Maximum cost threshold in USD.",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        help="Show detailed progress information.",
+    ),
+) -> None:
+    """Translate a document using LLMs."""
+    try:
+        # Parse model specification
+        model_type, model_name = parse_model_spec(model)
+
+        # Determine file type
+        file_type = FileType(input_file.suffix.lstrip(".").lower())
+
+        # Get cost estimate
+        estimate = estimate_cost(
+            input_file,
+            model_type,
+            max_cost=max_cost,
+        )
+
+        # Display cost estimate
+        console.print("\n[bold]Translation Plan[/bold]")
+        display_cost_estimate(estimate, model_type)
+
+        # In dry-run mode, exit after showing estimate
+        if dry_run:
+            return
+
+        # Check for warnings and confirm if needed
+        if not force and estimate.warnings:
+            console.print(
+                "\n[yellow]Warning:[/yellow] This translation has potential issues."
+            )
+            proceed = typer.confirm("Do you want to proceed?")
+            if not proceed:
+                console.print("\nTranslation cancelled.")
+                raise typer.Exit(1)
+
+        # Create translation config
+        config = TranslationConfig(
+            source_lang=source_lang or "auto",
+            target_lang=target_lang,
+            model=model_type,
+            model_name=model_name,
+            algorithm=algorithm,
+            input_file=input_file,
+            output_file=output_file,
+            force=force,
+            max_cost=max_cost,
+            verbose=verbose,
+        )
+
+        # Load document
+        content = asyncio.run(load_document(input_file))
+
+        # Initialize model interface
+        translator = create_translator(config)
+
+        # Show progress
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            console=console,
+        ) as progress:
+            # Run translation
+            response = asyncio.run(
+                translate_document(
+                    content=content,
+                    config=config,
+                    translator=translator,
+                    progress=progress,
+                )
+            )
+
+            # Convert TranslationResponse to TranslationResult
+            result = TranslationResult(
+                text=response.text,
+                tokens_used=response.tokens_used,
+                cost=response.cost,
+                time_taken=response.time_taken,
+            )
+
+        # Create translation output with metadata
+        output = TranslationOutput(
+            metadata=TranslationMetadata(
+                source_lang=config.source_lang,
+                target_lang=config.target_lang,
+                model=config.model,
+                algorithm=config.algorithm,
+                input_file=config.input_file,
+                input_file_type=file_type,
+            ),
+            result=result,
+            warnings=estimate.warnings,
+        )
+
+        # Get appropriate output handler
+        handler = create_handler(output_format)
+        handler.write(output, output_file)
+
+        # Show final statistics (only for text output)
+        if output_format == OutputFormat.TEXT:
+            table = Table(title="Translation Statistics", show_header=False)
+            table.add_column("Metric", style="bold blue")
+            table.add_column("Value")
+
+            table.add_row("Time Taken", f"{result.time_taken:.1f} seconds")
+            table.add_row("Tokens Used", f"{result.tokens_used:,}")
+            if model_type != ModelType.OLLAMA:
+                table.add_row("Final Cost", f"${result.cost:.4f}")
+
+            console.print("\n", table)
+
+    except Exception as e:
+        logger.exception("Translation failed")
+        console.print(f"[red]Error: {str(e)}[/red]")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    app()

tinbox/core/__init__.py

@@ -0,0 +1,26 @@
+"""Core functionality for Tinbox."""
+
+from tinbox.core.types import (
+    FileType,
+    ModelType,
+    TranslationConfig,
+    TranslationResult,
+)
+from tinbox.core.processor import DocumentContent, load_document
+from tinbox.core.translation import (
+    ModelInterface,
+    TranslationRequest,
+    TranslationResponse,
+    create_translator,
+)
+from tinbox.core.translation.algorithms import translate_document
+
+__all__ = [
+    "FileType",
+    "ModelType",
+    "TranslationConfig",
+    "TranslationResult",
+    "DocumentContent",
+    "load_document",
+    "translate_document",
+]

tinbox/core/cost.py

@@ -0,0 +1,158 @@
+"""Cost estimation utilities."""
+
+from enum import Enum
+from pathlib import Path
+from typing import Dict, Optional
+
+from tinbox.core.types import FileType, ModelType
+
+
+class CostLevel(str, Enum):
+    """Cost level classification."""
+
+    LOW = "low"  # < $1
+    MEDIUM = "medium"  # $1-$5
+    HIGH = "high"  # $5-$20
+    VERY_HIGH = "very_high"  # > $20
+
+
+# Approximate costs per 1K tokens (as of March 2024)
+MODEL_COSTS: Dict[ModelType, float] = {
+    ModelType.OPENAI: 0.03,  # $0.03 per 1K input tokens, $0.06 per 1K output tokens
+    ModelType.ANTHROPIC: 0.003,  # $0.003 per 1K input tokens, $0.015 per 1K output tokens
+    ModelType.OLLAMA: 0.0,  # Free for local models
+}
+
+
+def estimate_document_tokens(file_path: Path) -> int:
+    """Estimate the number of tokens in a document.
+
+    Args:
+        file_path: Path to the document
+
+    Returns:
+        Estimated number of tokens
+
+    Note:
+        These are rough estimates:
+        - PDF: 500 tokens per page
+        - DOCX: 1.3 tokens per word, rounded up
+        - TXT: 1 token per 4 characters, rounded up
+    """
+    file_type = FileType(file_path.suffix.lstrip(".").lower())
+
+    # Read sample of document to estimate tokens
+    if file_type == FileType.PDF:
+        # For PDFs, estimate based on number of pages
+        import pypdf
+
+        with open(file_path, "rb") as f:
+            pdf = pypdf.PdfReader(f)
+            # Rough estimate: 500 tokens per page
+            return len(pdf.pages) * 500
+    elif file_type == FileType.DOCX:
+        # For DOCX, estimate based on word count
+        from docx import Document
+
+        doc = Document(file_path)
+        word_count = sum(len(p.text.split()) for p in doc.paragraphs)
+        # Rough estimate: 1.3 tokens per word, rounded up
+        return int(word_count * 1.3 + 0.999)  # Round up by adding 0.999
+    else:  # TXT
+        # For text files, estimate based on character count
+        text = file_path.read_text()
+        # Rough estimate: 1 token per 4 characters, rounded up
+        return -(-len(text) // 4)  # Ceiling division
+
+
+def get_cost_level(cost: float) -> CostLevel:
+    """Get the cost level classification.
+
+    Args:
+        cost: Estimated cost in USD
+
+    Returns:
+        Cost level classification
+    """
+    if cost < 1.0:
+        return CostLevel.LOW
+    elif cost < 5.0:
+        return CostLevel.MEDIUM
+    elif cost < 20.0:
+        return CostLevel.HIGH
+    else:
+        return CostLevel.VERY_HIGH
+
+
+class CostEstimate:
+    """Cost estimate for a translation task."""
+
+    def __init__(
+        self,
+        estimated_tokens: int,
+        estimated_cost: float,
+        estimated_time: float,
+        warnings: list[str],
+    ) -> None:
+        """Initialize cost estimate.
+
+        Args:
+            estimated_tokens: Estimated number of tokens
+            estimated_cost: Estimated cost in USD
+            estimated_time: Estimated time in seconds
+            warnings: List of warning messages
+        """
+        self.estimated_tokens = estimated_tokens
+        self.estimated_cost = estimated_cost
+        self.estimated_time = estimated_time
+        self.warnings = warnings
+        self.cost_level = get_cost_level(estimated_cost)
+
+
+def estimate_cost(
+    file_path: Path,
+    model: ModelType,
+    *,
+    max_cost: Optional[float] = None,
+) -> CostEstimate:
+    """Estimate the cost of translating a document.
+
+    Args:
+        file_path: Path to the document
+        model: Model to use for translation
+        max_cost: Optional maximum cost threshold
+
+    Returns:
+        CostEstimate object with token count, cost, and warnings
+    """
+    estimated_tokens = estimate_document_tokens(file_path)
+    cost_per_1k = MODEL_COSTS.get(model, 0.0)
+    estimated_cost = (estimated_tokens / 1000) * cost_per_1k
+
+    # Estimate time (very rough estimate)
+    # Assume 5 tokens/second for cloud models, 20 tokens/second for local
+    tokens_per_second = 20 if model == ModelType.OLLAMA else 5
+    estimated_time = estimated_tokens / tokens_per_second
+
+    warnings = []
+
+    # Generate warnings
+    if model != ModelType.OLLAMA:
+        if estimated_tokens > 50000:  # More than 50K tokens
+            warnings.append(
+                f"Large document detected ({estimated_tokens:,} tokens). "
+                "Consider using Ollama for better performance and no cost."
+            )
+
+        if max_cost and estimated_cost > max_cost:
+            warnings.append(
+                f"Estimated cost (${estimated_cost:.2f}) exceeds maximum "
+                f"threshold (${max_cost:.2f})"
+            )
+
+    return CostEstimate(
+        estimated_tokens=estimated_tokens,
+        estimated_cost=estimated_cost,
+        estimated_time=estimated_time,
+        warnings=warnings,
+    )

tinbox/core/output.py

@@ -0,0 +1,212 @@
+"""Output handlers for different formats."""
+
+import json
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, Optional, Protocol, Union
+
+from pydantic import BaseModel, Field
+
+from tinbox.core.types import FileType, ModelType, TranslationResult
+from tinbox.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class OutputFormat(str, Enum):
+    """Supported output formats."""
+
+    TEXT = "text"
+    JSON = "json"
+    MARKDOWN = "markdown"
+
+
+class TranslationMetadata(BaseModel):
+    """Metadata about the translation process."""
+
+    source_lang: str
+    target_lang: str
+    model: ModelType
+    algorithm: str
+    input_file: Path
+    input_file_type: FileType
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class TranslationOutput(BaseModel):
+    """Complete translation output including metadata."""
+
+    metadata: TranslationMetadata
+    result: TranslationResult
+    warnings: list[str] = Field(default_factory=list)
+    errors: list[str] = Field(default_factory=list)
+
+
+class OutputHandler(Protocol):
+    """Protocol for output handlers."""
+
+    def write(
+        self,
+        output: TranslationOutput,
+        file: Optional[Path] = None,
+    ) -> None:
+        """Write the translation output.
+
+        Args:
+            output: The translation output to write
+            file: Optional file to write to. If None, writes to stdout.
+        """
+        ...
+
+
+class JSONOutputHandler:
+    """Handler for JSON output format."""
+
+    def write(
+        self,
+        output: TranslationOutput,
+        file: Optional[Path] = None,
+    ) -> None:
+        """Write translation output as JSON.
+
+        Args:
+            output: The translation output to write
+            file: Optional file to write to. If None, writes to stdout.
+        """
+        # Convert to JSON-serializable dict
+        data = output.model_dump(
+            mode="json",
+            exclude_none=True,
+        )
+
+        # Convert Path objects to strings
+        data["metadata"]["input_file"] = str(data["metadata"]["input_file"])
+        if "output_file" in data["metadata"]:
+            data["metadata"]["output_file"] = str(data["metadata"]["output_file"])
+
+        # Format JSON with indentation
+        json_str = json.dumps(data, indent=2)
+
+        if file:
+            file.write_text(json_str)
+        else:
+            print(json_str)
+
+
+class TextOutputHandler:
+    """Handler for plain text output format."""
+
+    def write(
+        self,
+        output: TranslationOutput,
+        file: Optional[Path] = None,
+    ) -> None:
+        """Write translation output as plain text.
+
+        Args:
+            output: The translation output to write
+            file: Optional file to write to. If None, writes to stdout.
+        """
+        # Just write the translated text
+        if file:
+            file.write_text(output.result.text)
+        else:
+            print(output.result.text)
+
+
+class MarkdownOutputHandler:
+    """Handler for Markdown output format."""
+
+    def write(
+        self,
+        output: TranslationOutput,
+        file: Optional[Path] = None,
+    ) -> None:
+        """Write translation output as Markdown.
+
+        Args:
+            output: The translation output to write
+            file: Optional file to write to. If None, writes to stdout.
+        """
+        # Build Markdown content
+        md_lines = [
+            "# Translation Results\n",
+            "## Metadata",
+            f"- Source Language: {output.metadata.source_lang}",
+            f"- Target Language: {output.metadata.target_lang}",
+            f"- Model: {output.metadata.model.value}",
+            f"- Algorithm: {output.metadata.algorithm}",
+            f"- Input File: {output.metadata.input_file.name}",
+            f"- File Type: {output.metadata.input_file_type.value}",
+            f"- Timestamp: {output.metadata.timestamp.isoformat()}\n",
+            "## Translation",
+            "```text",
+            output.result.text,
+            "```\n",
+            "## Statistics",
+            f"- Tokens Used: {output.result.tokens_used:,}",
+            f"- Cost: ${output.result.cost:.4f}",
+            f"- Time Taken: {output.result.time_taken:.1f}s\n",
+        ]
+
+        # Add warnings if any
+        if output.warnings:
+            md_lines.extend(
+                [
+                    "## Warnings",
+                    *[f"- {warning}" for warning in output.warnings],
+                    "",
+                ]
+            )
+
+        # Add errors if any
+        if output.errors:
+            md_lines.extend(
+                [
+                    "## Errors",
+                    *[f"- {error}" for error in output.errors],
+                    "",
+                ]
+            )
+        else:
+            md_lines.extend(
+                [
+                    "## Errors",
+                    "[None]",
+                    "",
+                ]
+            )
+
+        # Join lines with newlines
+        md_content = "\n".join(md_lines)
+
+        if file:
+            file.write_text(md_content)
+        else:
+            print(md_content)
+
+
+def create_handler(format: OutputFormat) -> OutputHandler:
+    """Create an output handler for the specified format.
+
+    Args:
+        format: The desired output format
+
+    Returns:
+        An appropriate output handler
+
+    Raises:
+        ValueError: If the format is not supported
+    """
+    handlers = {
+        OutputFormat.TEXT: TextOutputHandler(),
+        OutputFormat.JSON: JSONOutputHandler(),
+        OutputFormat.MARKDOWN: MarkdownOutputHandler(),
+    }
+
+    handler = handlers.get(format)
+    if not handler:
+        raise ValueError(f"Unsupported output format: {format}")
+
+    return handler