ostruct-cli 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

ostruct/cli/template_rendering.py

@@ -64,7 +64,12 @@ from jinja2 import Environment
 from .errors import TaskTemplateVariableError, TemplateValidationError
 from .file_utils import FileInfo
 from .progress import ProgressContext
+from .progress_reporting import get_progress_reporter
 from .template_env import create_jinja_env
+from .template_optimizer import (
+    is_optimization_beneficial,
+    optimize_template_for_llm,
+)
 from .template_schema import DotDict, StdinProxy
 
 __all__ = [
@@ -162,20 +167,6 @@ def render_template(
             if "stdin" not in wrapped_context:
                 wrapped_context["stdin"] = StdinProxy()
 
-            # Load file content for FileInfo objects
-            for key, value in context.items():
-                if isinstance(value, FileInfo):
-                    # Access content property to trigger loading
-                    _ = value.content
-                elif (
-                    isinstance(value, list)
-                    and value
-                    and isinstance(value[0], FileInfo)
-                ):
-                    for file_info in value:
-                        # Access content property to trigger loading
-                        _ = file_info.content
-
             if progress:
                 progress.update(1)  # Update progress for template creation
 
@@ -235,24 +226,14 @@ def render_template(
                         )
                         if value and isinstance(value[0], FileInfo):
                             logger.info(
-                                "    First file: %s (content length: %d)",
+                                "    First file: %s",
                                 value[0].path,
-                                (
-                                    len(value[0].content)
-                                    if hasattr(value[0], "content")
-                                    else -1
-                                ),
                             )
                     elif isinstance(value, FileInfo):
                         logger.info(
-                            "  %s: FileInfo(%s) content length: %d",
+                            "  %s: FileInfo(%s)",
                             key,
                             value.path,
-                            (
-                                len(value.content)
-                                if hasattr(value, "content")
-                                else -1
-                            ),
                         )
                     else:
                         logger.info("  %s: %s", key, type(value).__name__)
@@ -278,18 +259,72 @@ def render_template(
                                     "      exists: %r",
                                     os.path.exists(item.path),
                                 )
-                                logger.debug(
-                                    "      content length: %d",
-                                    (
-                                        len(item.content)
-                                        if hasattr(item, "content")
-                                        else -1
-                                    ),
-                                )
                     else:
                         logger.debug(
                             "  %s: %s (%r)", key, type(value).__name__, value
                         )
+
+                # Apply template optimization for better LLM performance
+                try:
+                    # Get template source - use template_str for string templates or template.source for file templates
+                    if hasattr(template, "source") and template.source:
+                        original_template_source = template.source
+                    else:
+                        original_template_source = template_str
+
+                    if (
+                        original_template_source
+                        and is_optimization_beneficial(
+                            original_template_source
+                        )
+                    ):
+                        logger.debug("=== Template Optimization ===")
+                        optimization_result = optimize_template_for_llm(
+                            original_template_source
+                        )
+
+                        if optimization_result.has_optimizations:
+                            # Report optimization to user
+                            progress_reporter = get_progress_reporter()
+                            progress_reporter.report_optimization(
+                                optimization_result.transformations
+                            )
+
+                            logger.info(
+                                "Template optimized for LLM performance:"
+                            )
+                            for (
+                                transformation
+                            ) in optimization_result.transformations:
+                                logger.info(f"  • {transformation}")
+                            logger.info(
+                                f"  • Optimization time: {optimization_result.optimization_time_ms:.1f}ms"
+                            )
+
+                            # Create new template from optimized content
+                            template = env.from_string(
+                                optimization_result.optimized_template
+                            )
+                            # Re-add globals to new template
+                            template.globals["template_name"] = getattr(
+                                template, "name", "<string>"
+                            )
+                            template.globals["template_path"] = getattr(
+                                template, "filename", None
+                            )
+                        else:
+                            logger.debug("No beneficial optimizations found")
+                    else:
+                        logger.debug(
+                            "Template optimization not beneficial - skipping"
+                        )
+                except Exception as e:
+                    # If optimization fails, continue with original template
+                    logger.warning(
+                        f"Template optimization failed, using original: {e}"
+                    )
+                    # template remains unchanged
+
                 result = template.render(**wrapped_context)
                 if not isinstance(result, str):
                     raise TemplateValidationError(

ostruct/cli/token_validation.py

@@ -0,0 +1,286 @@
+"""Token limit validation with actionable error guidance."""
+
+import logging
+import os
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import tiktoken
+
+from .errors import PromptTooLargeError
+
+
+class TokenLimitValidator:
+    """Validate prompt size and provide corrective guidance for explicit file routing."""
+
+    MAX_TOKENS = 128000  # Model context window limit
+
+    def __init__(self, model: str = "gpt-4o"):
+        """Initialize validator with model-specific encoding.
+
+        Args:
+            model: Model name for token encoding selection
+        """
+        self.model = model
+        self.encoder = self._get_encoder(model)
+
+    def _get_encoder(self, model: str) -> tiktoken.Encoding:
+        """Get appropriate tiktoken encoder for model."""
+        if model.startswith(("gpt-4o", "o1", "o3")):
+            return tiktoken.get_encoding("o200k_base")
+        else:
+            return tiktoken.get_encoding("cl100k_base")
+
+    def validate_prompt_size(
+        self,
+        template_content: str,
+        template_files: List[str],
+        context_limit: Optional[int] = None,
+    ) -> None:
+        """Check if prompt will exceed context window and provide actionable guidance.
+
+        Args:
+            template_content: Rendered template content
+            template_files: List of file paths included in template
+            context_limit: Optional custom context limit (defaults to MAX_TOKENS)
+
+        Raises:
+            PromptTooLargeError: If prompt exceeds context window with actionable guidance
+        """
+        logger = logging.getLogger(__name__)
+
+        limit = context_limit or self.MAX_TOKENS
+        total_tokens = self._count_template_tokens(template_content)
+
+        oversized_files = []
+        for file_path in template_files:
+            try:
+                file_tokens = self._count_file_tokens(file_path)
+                total_tokens += file_tokens
+
+                # Flag files over 5K tokens for routing guidance
+                if file_tokens > 5000:
+                    oversized_files.append((file_path, file_tokens))
+            except (OSError, IOError):
+                # Skip files that can't be read for token counting
+                continue
+
+        # Add 90% warning threshold
+        if total_tokens > limit * 0.9:
+            logger.warning(
+                "Prompt is %.1f%% of the %d-token window (%d tokens)",
+                total_tokens / limit * 100,
+                limit,
+                total_tokens,
+            )
+
+        if total_tokens > limit:
+            self._raise_actionable_error(total_tokens, limit, oversized_files)
+
+    def _count_template_tokens(self, content: str) -> int:
+        """Count tokens in template content."""
+        return len(self.encoder.encode(content))
+
+    def _count_file_tokens(self, file_path: str) -> int:
+        """Count tokens in a file."""
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                content = f.read()
+                return len(self.encoder.encode(content))
+        except UnicodeDecodeError:
+            # For binary files, estimate based on file size
+            # Rough estimate: 1 token per 4 bytes
+            file_size = os.path.getsize(file_path)
+            return file_size // 4
+
+    def _is_data_file(self, file_path: str) -> bool:
+        """Detect if file is likely a data file suitable for Code Interpreter."""
+        data_extensions = {
+            ".csv",
+            ".json",
+            ".xlsx",
+            ".xls",
+            ".tsv",
+            ".parquet",
+            ".sql",
+            ".db",
+            ".sqlite",
+            ".sqlite3",
+            ".pkl",
+            ".pickle",
+            ".npy",
+            ".npz",
+            ".h5",
+            ".hdf5",
+            ".xml",
+            ".yaml",
+            ".yml",
+        }
+        return Path(file_path).suffix.lower() in data_extensions
+
+    def _is_document_file(self, file_path: str) -> bool:
+        """Detect if file is likely a document suitable for File Search."""
+        doc_extensions = {
+            ".pdf",
+            ".doc",
+            ".docx",
+            ".txt",
+            ".md",
+            ".rst",
+            ".tex",
+            ".html",
+            ".htm",
+            ".rtf",
+            ".odt",
+            ".epub",
+            ".mobi",
+        }
+        return Path(file_path).suffix.lower() in doc_extensions
+
+    def _is_code_file(self, file_path: str) -> bool:
+        """Detect if file is likely source code."""
+        code_extensions = {
+            ".py",
+            ".js",
+            ".ts",
+            ".java",
+            ".cpp",
+            ".c",
+            ".h",
+            ".hpp",
+            ".cs",
+            ".go",
+            ".rs",
+            ".rb",
+            ".php",
+            ".swift",
+            ".kt",
+            ".scala",
+            ".r",
+            ".m",
+            ".sh",
+            ".bash",
+            ".ps1",
+            ".pl",
+            ".lua",
+            ".dart",
+        }
+        return Path(file_path).suffix.lower() in code_extensions
+
+    def _raise_actionable_error(
+        self,
+        total_tokens: int,
+        limit: int,
+        oversized_files: List[Tuple[str, int]],
+    ) -> None:
+        """Raise PromptTooLargeError with specific guidance for explicit file routing."""
+        error_msg = (
+            f"❌ Error: Prompt exceeds model context window "
+            f"({total_tokens:,} tokens > {limit:,} limit)\n\n"
+        )
+
+        if oversized_files:
+            error_msg += "💡 Suggestion: Re-run with explicit file routing to move large files out of template context:\n\n"
+
+            for file_path, tokens in oversized_files:
+                file_name = Path(file_path).name
+
+                if self._is_data_file(file_path):
+                    error_msg += f"   📊 Data file: ostruct -fc {file_name} <template> <schema>\n"
+                    error_msg += f"       (Moves {file_name} to Code Interpreter for data processing)\n\n"
+                elif self._is_document_file(file_path):
+                    error_msg += f"   📄 Document: ostruct -fs {file_name} <template> <schema>\n"
+                    error_msg += f"       (Moves {file_name} to File Search for semantic retrieval)\n\n"
+                elif self._is_code_file(file_path):
+                    error_msg += f"   💻 Code file: ostruct -fc {file_name} <template> <schema>\n"
+                    error_msg += f"       (Moves {file_name} to Code Interpreter for analysis)\n\n"
+                else:
+                    error_msg += f"   📁 Large file: ostruct -fc {file_name} OR -fs {file_name} <template> <schema>\n"
+                    error_msg += "       (Choose based on usage: -fc for processing, -fs for retrieval)\n\n"
+
+                error_msg += (
+                    f"       Size: {tokens:,} tokens ({file_path})\n\n"
+                )
+
+            error_msg += (
+                "🔧 Alternative: Use --file-for for specific tool routing:\n"
+            )
+            error_msg += f"    ostruct --file-for code-interpreter {oversized_files[0][0]} <template> <schema>\n\n"
+
+        else:
+            error_msg += "💡 Suggestion: Consider breaking down your template or using fewer input files\n\n"
+
+        error_msg += "🔍 Check file sizes: tiktoken_cli count <filename>\n"
+        error_msg += "📖 Learn more: ostruct --help (see File Routing section)"
+
+        raise PromptTooLargeError(
+            error_msg,
+            context={
+                "total_tokens": total_tokens,
+                "context_limit": limit,
+                "oversized_files": [
+                    (path, tokens) for path, tokens in oversized_files
+                ],
+                "suggested_routing": self._generate_routing_suggestions(
+                    oversized_files
+                ),
+            },
+        )
+
+    def _generate_routing_suggestions(
+        self, oversized_files: List[Tuple[str, int]]
+    ) -> List[dict]:
+        """Generate structured routing suggestions for programmatic use."""
+        suggestions = []
+        for file_path, tokens in oversized_files:
+            suggestion = {
+                "file_path": file_path,
+                "tokens": tokens,
+                "file_type": self._classify_file(file_path),
+                "recommended_flags": self._get_recommended_flags(file_path),
+            }
+            suggestions.append(suggestion)
+        return suggestions
+
+    def _classify_file(self, file_path: str) -> str:
+        """Classify file type for routing suggestions."""
+        if self._is_data_file(file_path):
+            return "data"
+        elif self._is_document_file(file_path):
+            return "document"
+        elif self._is_code_file(file_path):
+            return "code"
+        else:
+            return "unknown"
+
+    def _get_recommended_flags(self, file_path: str) -> List[str]:
+        """Get recommended CLI flags for file routing."""
+        if self._is_data_file(file_path) or self._is_code_file(file_path):
+            return ["-fc", "--file-for code-interpreter"]
+        elif self._is_document_file(file_path):
+            return ["-fs", "--file-for file-search"]
+        else:
+            return ["-fc", "-fs"]  # Both options for unknown files
+
+
+def validate_token_limits(
+    template_content: str,
+    template_files: List[str],
+    model: str,
+    context_limit: Optional[int] = None,
+) -> None:
+    """Convenience function for token limit validation.
+
+    Args:
+        template_content: Rendered template content
+        template_files: List of file paths included in template
+        model: Model name for encoding selection
+        context_limit: Optional custom context limit
+
+    Raises:
+        PromptTooLargeError: If prompt exceeds context window
+    """
+    validator = TokenLimitValidator(model)
+    validator.validate_prompt_size(
+        template_content, template_files, context_limit
+    )

ostruct/cli/types.py

@@ -0,0 +1,78 @@
+"""Type definitions for ostruct CLI."""
+
+from pathlib import Path
+from typing import List, Optional, Tuple, TypedDict, Union
+
+# Import FileRoutingResult from validators
+FileRoutingResult = List[Tuple[Optional[str], Union[str, Path]]]
+
+
+class CLIParams(TypedDict, total=False):
+    """Type-safe CLI parameters."""
+
+    files: List[
+        Tuple[str, str]
+    ]  # List of (name, path) tuples from Click's nargs=2
+    dir: List[
+        Tuple[str, str]
+    ]  # List of (name, dir) tuples from Click's nargs=2
+    patterns: List[
+        Tuple[str, str]
+    ]  # List of (name, pattern) tuples from Click's nargs=2
+    allowed_dirs: List[str]
+    base_dir: str
+    allowed_dir_file: Optional[str]
+    recursive: bool
+    var: List[str]
+    json_var: List[str]
+    system_prompt: Optional[str]
+    system_prompt_file: Optional[str]
+    ignore_task_sysprompt: bool
+    model: str
+    timeout: float
+    output_file: Optional[str]
+    dry_run: bool
+    no_progress: bool
+    api_key: Optional[str]
+    verbose: bool
+    debug_openai_stream: bool
+    show_model_schema: bool
+    debug_validation: bool
+    temperature: Optional[float]
+    max_output_tokens: Optional[int]
+    top_p: Optional[float]
+    frequency_penalty: Optional[float]
+    presence_penalty: Optional[float]
+    reasoning_effort: Optional[str]
+    progress_level: str
+    task_file: Optional[str]
+    task: Optional[str]
+    schema_file: str
+    mcp_servers: List[str]
+    mcp_allowed_tools: List[str]
+    mcp_require_approval: str
+    mcp_headers: Optional[str]
+    code_interpreter_files: FileRoutingResult  # Fixed: was List[str]
+    code_interpreter_dirs: List[str]
+    code_interpreter_download_dir: str
+    code_interpreter_cleanup: bool
+    file_search_files: FileRoutingResult  # Fixed: was List[str]
+    file_search_dirs: List[str]
+    file_search_vector_store_name: str
+    file_search_cleanup: bool
+    file_search_retry_count: int
+    file_search_timeout: float
+    template_files: FileRoutingResult  # Fixed: was List[str]
+    template_dirs: List[str]
+    template_file_aliases: List[
+        Tuple[str, Union[str, Path]]
+    ]  # Fixed: was List[Tuple[str, str]]
+    code_interpreter_file_aliases: List[
+        Tuple[str, Union[str, Path]]
+    ]  # Fixed: was List[Tuple[str, str]]
+    file_search_file_aliases: List[
+        Tuple[str, Union[str, Path]]
+    ]  # Fixed: was List[Tuple[str, str]]
+    tool_files: List[
+        Tuple[str, str]
+    ]  # List of (tool, path) tuples from --file-for