ostruct-cli 0.8.29__py3-none-any.whl → 1.0.1__py3-none-any.whl

ostruct/cli/template_rendering.py

@@ -65,12 +65,7 @@ from jinja2 import Environment
 from .errors import TaskTemplateVariableError, TemplateValidationError
 from .file_utils import FileInfo
 from .progress import ProgressContext
-from .progress_reporting import get_progress_reporter
 from .template_env import create_jinja_env
-from .template_optimizer import (
-    is_optimization_beneficial,
-    optimize_template_for_llm,
-)
 from .template_schema import DotDict, StdinProxy
 
 __all__ = [
@@ -156,7 +151,7 @@ def render_template(
         TaskTemplateVariableError: If template variables are undefined
         TemplateValidationError: If template rendering fails for other reasons
     """
-    from .progress import (  # Import here to avoid circular dependency
+    from .progress import (
         ProgressContext,
     )
 
@@ -169,7 +164,7 @@ def render_template(
                 progress.update(1)  # Update progress for setup
 
             if env is None:
-                env = create_jinja_env(loader=jinja2.FileSystemLoader("."))
+                env, _ = create_jinja_env(loader=jinja2.FileSystemLoader("."))
 
             logger.debug("=== Raw Input ===")
             logger.debug(
@@ -304,67 +299,6 @@ def render_template(
                             "  %s: %s (%r)", key, type(value).__name__, value
                         )
 
-                # Apply template optimization for better LLM performance
-                try:
-                    # Get template source - use template_str for string templates or template.source for file templates
-                    if hasattr(template, "source") and template.source:
-                        original_template_source = template.source
-                    else:
-                        original_template_source = template_str
-
-                    if (
-                        original_template_source
-                        and is_optimization_beneficial(
-                            original_template_source
-                        )
-                    ):
-                        logger.debug("=== Template Optimization ===")
-                        optimization_result = optimize_template_for_llm(
-                            original_template_source
-                        )
-
-                        if optimization_result.has_optimizations:
-                            # Report optimization to user
-                            progress_reporter = get_progress_reporter()
-                            progress_reporter.report_optimization(
-                                optimization_result.transformations
-                            )
-
-                            logger.info(
-                                "Template optimized for LLM performance:"
-                            )
-                            for (
-                                transformation
-                            ) in optimization_result.transformations:
-                                logger.info(f"  • {transformation}")
-                            logger.info(
-                                f"  • Optimization time: {optimization_result.optimization_time_ms:.1f}ms"
-                            )
-
-                            # Create new template from optimized content
-                            template = env.from_string(
-                                optimization_result.optimized_template
-                            )
-                            # Re-add globals to new template
-                            template.globals["template_name"] = getattr(
-                                template, "name", "<string>"
-                            )
-                            template.globals["template_path"] = getattr(
-                                template, "filename", None
-                            )
-                        else:
-                            logger.debug("No beneficial optimizations found")
-                    else:
-                        logger.debug(
-                            "Template optimization not beneficial - skipping"
-                        )
-                except Exception as e:
-                    # If optimization fails, continue with original template
-                    logger.warning(
-                        f"Template optimization failed, using original: {e}"
-                    )
-                    # template remains unchanged
-
                 result = template.render(**wrapped_context)
                 if not isinstance(result, str):
                     raise TemplateValidationError(
@@ -391,11 +325,53 @@ def render_template(
                     f"  -V {var_name}='value'"
                 )
                 raise TaskTemplateVariableError(error_msg) from e
-            except (jinja2.TemplateError, Exception) as e:
-                logger.error("Template rendering failed: %s", str(e))
-                raise TemplateValidationError(
-                    f"Template rendering failed: {str(e)}"
-                ) from e
+            except TypeError as e:
+                error_str = str(e)
+                # Handle iteration errors with user-friendly messages
+                if "object is not iterable" in error_str:
+                    # Extract variable name from iteration context
+                    # Look for patterns like "'ClassName' object is not iterable"
+                    # and provide helpful guidance
+                    user_friendly_msg = (
+                        "Template iteration error: A variable used in a loop ({% for ... %}) "
+                        "is not iterable. This usually means:\n"
+                        "1. The variable is not a file object, list, or other iterable type\n"
+                        "2. Check that your template variable names match the expected data\n"
+                        "3. Verify the variable contains the expected data type\n"
+                        f"Available variables: {', '.join(sorted(context.keys()))}"
+                    )
+                    logger.error("Template iteration error: %s", error_str)
+                    raise TemplateValidationError(user_friendly_msg) from e
+                else:
+                    # Other TypeError - preserve original behavior
+                    logger.error("Template rendering failed: %s", str(e))
+                    raise TemplateValidationError(
+                        f"Template rendering failed: {str(e)}"
+                    ) from e
+            except Exception as e:
+                # Import here to avoid circular imports
+                from .template_filters import (
+                    TemplateStructureError,
+                    format_tses_error,
+                )
+
+                # Handle TemplateStructureError with helpful formatting
+                if isinstance(e, TemplateStructureError):
+                    formatted_error = format_tses_error(e)
+                    logger.error(
+                        "Template structure error: %s", formatted_error
+                    )
+                    raise TemplateValidationError(formatted_error) from e
+                elif isinstance(e, jinja2.TemplateError):
+                    logger.error("Template rendering failed: %s", str(e))
+                    raise TemplateValidationError(
+                        f"Template rendering failed: {str(e)}"
+                    ) from e
+                else:
+                    logger.error("Template rendering failed: %s", str(e))
+                    raise TemplateValidationError(
+                        f"Template rendering failed: {str(e)}"
+                    ) from e
 
         except ValueError as e:
             # Re-raise with original context

ostruct/cli/template_validation.py

@@ -218,7 +218,8 @@ def validate_template_placeholders(
 
     try:
         # 1) Create Jinja2 environment with meta extension and safe undefined
-        env = create_jinja_env(validation_mode=True)
+        env_tuple = create_jinja_env(validation_mode=True)
+        env = env_tuple[0]
 
         # Register custom filters with None-safe wrappers
         env.filters.update(

ostruct/cli/token_validation.py

@@ -184,19 +184,36 @@ class TokenLimitValidator:
 
             for file_path, tokens in oversized_files:
                 file_name = Path(file_path).name
-
-                if self._is_data_file(file_path):
-                    error_msg += f"   📊 Data file: ostruct -fc {file_name} <template> <schema>\n"
-                    error_msg += f"       (Moves {file_name} to Code Interpreter for data processing)\n\n"
-                elif self._is_document_file(file_path):
-                    error_msg += f"   📄 Document: ostruct -fs {file_name} <template> <schema>\n"
-                    error_msg += f"       (Moves {file_name} to File Search for semantic retrieval)\n\n"
-                elif self._is_code_file(file_path):
-                    error_msg += f"   💻 Code file: ostruct -fc {file_name} <template> <schema>\n"
-                    error_msg += f"       (Moves {file_name} to Code Interpreter for analysis)\n\n"
+                file_extension = Path(file_path).suffix.lower()
+
+                if file_extension in [
+                    ".csv",
+                    ".json",
+                    ".xml",
+                    ".yaml",
+                    ".yml",
+                ]:
+                    error_msg += f"   📊 Data file: ostruct --file ci:data {file_name} <template> <schema>\n"
+                elif file_extension in [
+                    ".pdf",
+                    ".txt",
+                    ".md",
+                    ".doc",
+                    ".docx",
+                ]:
+                    error_msg += f"   📄 Document: ostruct --file fs:docs {file_name} <template> <schema>\n"
+                elif file_extension in [
+                    ".py",
+                    ".js",
+                    ".ts",
+                    ".java",
+                    ".cpp",
+                    ".c",
+                ]:
+                    error_msg += f"   💻 Code file: ostruct --file ci:code {file_name} <template> <schema>\n"
                 else:
-                    error_msg += f"   📁 Large file: ostruct -fc {file_name} OR -fs {file_name} <template> <schema>\n"
-                    error_msg += "       (Choose based on usage: -fc for processing, -fs for retrieval)\n\n"
+                    error_msg += f"   📁 Large file: ostruct --file ci:data {file_name} OR --file fs:docs {file_name} <template> <schema>\n"
+                    error_msg += "       (Choose based on usage: ci for processing, fs for retrieval)\n\n"
 
                 error_msg += (
                     f"       Size: {tokens:,} tokens ({file_path})\n\n"
@@ -256,11 +273,14 @@ class TokenLimitValidator:
     def _get_recommended_flags(self, file_path: str) -> List[str]:
         """Get recommended CLI flags for file routing."""
         if self._is_data_file(file_path) or self._is_code_file(file_path):
-            return ["-fc", "--file-for code-interpreter"]
+            return ["--file ci:data", "--file ci:code"]
         elif self._is_document_file(file_path):
-            return ["-fs", "--file-for file-search"]
+            return ["--file fs:docs", "--file fs:knowledge"]
         else:
-            return ["-fc", "-fs"]  # Both options for unknown files
+            return [
+                "--file ci:data",
+                "--file fs:docs",
+            ]  # Both options for unknown files
 
 
 def validate_token_limits(

ostruct/cli/types.py

@@ -1,7 +1,7 @@
 """Type definitions for ostruct CLI."""
 
 from pathlib import Path
-from typing import List, Optional, Tuple, TypedDict, Union
+from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union
 
 # Import FileRoutingResult from validators
 FileRoutingResult = List[Tuple[Optional[str], Union[str, Path]]]
@@ -32,7 +32,6 @@ class CLIParams(TypedDict, total=False):
     timeout: float
     output_file: Optional[str]
     dry_run: bool
-    no_progress: bool
     api_key: Optional[str]
     verbose: bool
     show_model_schema: bool
@@ -43,24 +42,30 @@ class CLIParams(TypedDict, total=False):
     frequency_penalty: Optional[float]
     presence_penalty: Optional[float]
     reasoning_effort: Optional[str]
-    progress_level: str
+    progress: str
     task_file: Optional[str]
     task: Optional[str]
     schema_file: str
     mcp_servers: List[str]
+
+    # New attachment system (T3.0)
+    attaches: List[Dict[str, Any]]  # --attach specifications
+    dirs: List[Dict[str, Any]]  # --dir specifications
+    collects: List[Dict[str, Any]]  # --collect specifications
     mcp_allowed_tools: List[str]
     mcp_require_approval: str
     mcp_headers: Optional[str]
     code_interpreter_files: FileRoutingResult  # Fixed: was List[str]
     code_interpreter_dirs: List[str]
-    code_interpreter_download_dir: str
-    code_interpreter_cleanup: bool
+    ci_download_dir: str
+    ci_duplicate_outputs: Optional[str]
+    ci_cleanup: bool
     file_search_files: FileRoutingResult  # Fixed: was List[str]
     file_search_dirs: List[str]
-    file_search_vector_store_name: str
-    file_search_cleanup: bool
-    file_search_retry_count: int
-    file_search_timeout: float
+    fs_store_name: str
+    fs_cleanup: bool
+    fs_retries: int
+    fs_timeout: float
     template_files: FileRoutingResult  # Fixed: was List[str]
     template_dirs: List[str]
     template_file_aliases: List[
@@ -75,17 +80,8 @@ class CLIParams(TypedDict, total=False):
     tool_files: List[
         Tuple[str, str]
     ]  # List of (tool, path) tuples from --file-for
-    web_search: bool
     debug: bool
-    show_templates: bool
-    debug_templates: bool
-    show_context: bool
-    show_context_detailed: bool
-    show_pre_optimization: bool
-    show_optimization_diff: bool
-    no_optimization: bool
-    show_optimization_steps: bool
-    optimization_step_detail: str
+
     help_debug: bool
     enabled_features: List[str]  # List of feature names to enable
     disabled_features: List[str]  # List of feature names to disable

ostruct/cli/unicode_compat.py

@@ -0,0 +1,283 @@
+"""Windows Unicode compatibility utilities.
+
+This module provides smart emoji handling for Windows terminals:
+- Modern terminals (Windows Terminal, PowerShell, Win11 Console) get full emoji
+- Legacy terminals (cmd.exe with cp1252) get clean text without emoji
+- Detection is automatic and graceful with user overrides available
+
+Environment Variables:
+- OSTRUCT_UNICODE=auto: Auto-detect terminal capabilities (default)
+- OSTRUCT_UNICODE=1/true/yes: Force emoji display regardless of detection
+- OSTRUCT_UNICODE=0/false/no: Force plain text regardless of detection
+- OSTRUCT_UNICODE=debug: Show detection details and use auto-detection
+"""
+
+import os
+import sys
+from typing import Any, Optional
+
+
+def _get_unicode_setting() -> str:
+    """Get the Unicode setting from environment variable."""
+    return os.environ.get("OSTRUCT_UNICODE", "auto").lower()
+
+
+def _is_debug_mode() -> bool:
+    """Check if Unicode debugging is enabled."""
+    return _get_unicode_setting() == "debug"
+
+
+def _debug_log(message: str) -> None:
+    """Log debug message if unicode debugging is enabled."""
+    if _is_debug_mode():
+        print(f"[UNICODE DEBUG] {message}", file=sys.stderr)
+
+
+def _detect_modern_windows_terminal() -> bool:
+    """Detect if we're running in a modern Windows terminal that supports Unicode.
+
+    Returns:
+        True if running in Windows Terminal, PowerShell, or Win11 Console Host
+        False if running in legacy cmd.exe or other limited terminals
+    """
+    # Check for explicit user overrides first
+    unicode_setting = _get_unicode_setting()
+
+    if unicode_setting in ("1", "true", "yes", "on"):
+        _debug_log("Unicode forced ON via OSTRUCT_UNICODE")
+        return True
+
+    if unicode_setting in ("0", "false", "no", "off"):
+        _debug_log("Unicode forced OFF via OSTRUCT_UNICODE")
+        return False
+
+    # For "auto" and "debug", proceed with detection
+
+    if not sys.platform.startswith("win"):
+        # Non-Windows systems generally support Unicode well
+        _debug_log(f"Non-Windows platform ({sys.platform}): Unicode enabled")
+        return True
+
+    _debug_log("Windows platform detected, checking terminal capabilities...")
+
+    # Check for Windows Terminal
+    if os.environ.get("WT_SESSION"):
+        _debug_log("Windows Terminal detected via WT_SESSION")
+        return True
+
+    # Check for PowerShell (both Windows PowerShell and PowerShell Core)
+    if os.environ.get("PSModulePath"):
+        _debug_log("PowerShell detected via PSModulePath")
+        return True
+
+    # Check for modern console host (Windows 11+)
+    # The TERMINAL_EMULATOR environment variable is set by modern terminals
+    if os.environ.get("TERMINAL_EMULATOR"):
+        _debug_log(
+            f"Modern terminal detected via TERMINAL_EMULATOR: {os.environ.get('TERMINAL_EMULATOR')}"
+        )
+        return True
+
+    # Check for VS Code integrated terminal
+    if os.environ.get("VSCODE_INJECTION"):
+        _debug_log("VS Code integrated terminal detected")
+        return True
+
+    # Check for GitHub Codespaces
+    if os.environ.get("CODESPACES"):
+        _debug_log("GitHub Codespaces detected")
+        return True
+
+    # Check console code page - UTF-8 indicates modern setup
+    try:
+        import locale
+
+        encoding = locale.getpreferredencoding()
+        if encoding.lower() in ("utf-8", "utf8"):
+            _debug_log(f"UTF-8 locale detected: {encoding}")
+            return True
+        else:
+            _debug_log(f"Non-UTF-8 locale: {encoding}")
+    except Exception as e:
+        _debug_log(f"Locale detection failed: {e}")
+
+    # Check if stdout encoding supports Unicode
+    try:
+        stdout_encoding = getattr(sys.stdout, "encoding", None)
+        if stdout_encoding and stdout_encoding.lower() in ("utf-8", "utf8"):
+            _debug_log(f"UTF-8 stdout encoding: {stdout_encoding}")
+            return True
+        else:
+            _debug_log(f"Non-UTF-8 stdout encoding: {stdout_encoding}")
+    except Exception as e:
+        _debug_log(f"Stdout encoding detection failed: {e}")
+
+    # Check Windows version (Windows 10 1903+ has better Unicode support)
+    try:
+        import platform
+
+        version = platform.version()
+        # Windows 10 build 18362 (1903) and later have better Unicode support
+        if version and "10.0." in version:
+            build = version.split(".")[-1]
+            if build.isdigit() and int(build) >= 18362:
+                _debug_log(f"Modern Windows version detected: {version}")
+                return True
+        _debug_log(f"Windows version: {version}")
+    except Exception as e:
+        _debug_log(f"Windows version detection failed: {e}")
+
+    # Default to False for safety on Windows
+    _debug_log(
+        "No modern terminal indicators found, defaulting to legacy mode"
+    )
+    return False
+
+
+def safe_emoji(emoji: str, text_without_emoji: Optional[str] = None) -> str:
+    """Return emoji on Unicode-capable terminals, clean text on legacy terminals.
+
+    Args:
+        emoji: The emoji character to display
+        text_without_emoji: Optional clean text version. If None, emoji is simply omitted.
+
+    Returns:
+        Emoji on modern terminals, clean text on legacy terminals
+
+    Examples:
+        safe_emoji("🚀", "START") -> "🚀" or "START"
+        safe_emoji("🔍") -> "🔍" or ""
+    """
+    if _detect_modern_windows_terminal():
+        # Modern terminal - try to use emoji
+        try:
+            # Test if we can encode the emoji
+            emoji.encode(sys.stdout.encoding or "utf-8")
+            return emoji
+        except (UnicodeEncodeError, LookupError, AttributeError):
+            # Fallback even on modern terminals if encoding fails
+            return text_without_emoji or ""
+    else:
+        # Legacy terminal - use clean text
+        return text_without_emoji or ""
+
+
+def safe_format(format_string: str, *args: Any, **kwargs: Any) -> str:
+    """Format string with emoji safety.
+
+    Processes format strings containing emoji through safe_emoji() automatically.
+    Optimized to skip processing when no emoji are present.
+
+    Args:
+        format_string: Format string that may contain emoji
+        *args, **kwargs: Format arguments
+
+    Returns:
+        Formatted string with emoji handled appropriately for the terminal
+    """
+    # Quick check: if no emoji characters are present, skip processing entirely
+    # This optimizes the common case of plain text messages
+    # Check for the specific emoji we handle rather than broad Unicode ranges
+    emoji_chars = {
+        "🚀",
+        "🔍",
+        "⚙️",
+        "ℹ️",
+        "📖",
+        "💻",
+        "📄",
+        "🌐",
+        "🕐",
+        "📋",
+        "🤖",
+        "🔒",
+        "🛠️",
+        "📎",
+        "📥",
+        "📊",
+        "💰",
+        "✅",
+        "❌",
+        "⚠️",
+        "⏱️",
+    }
+    if not any(emoji in format_string for emoji in emoji_chars):
+        return format_string.format(*args, **kwargs)
+
+    # Common emoji replacements for CLI output
+    emoji_map = {
+        "🚀": "",  # Just omit, the text context is clear
+        "🔍": "",  # Just omit, "Plan" or "Execution Plan" is clear
+        "⚙️": "",  # Just omit for progress reporting
+        "ℹ️": "",  # Just omit for info messages
+        "📖": "",  # Just omit for help references
+        "💻": "",  # Just omit for Code Interpreter
+        "📄": "",  # Just omit for file references
+        "🌐": "",  # Just omit for web search
+        "🕐": "",  # Just omit for timestamp
+        "📋": "",  # Just omit for schema
+        "🤖": "",  # Just omit for model
+        "🔒": "",  # Just omit for security
+        "🛠️": "",  # Just omit for tools
+        "📎": "",  # Just omit for attachments
+        "📥": "",  # Just omit for downloads
+        "📊": "",  # Just omit for variables
+        "💰": "",  # Just omit for cost
+        "✅": "[OK]",  # Show status indicator
+        "❌": "[ERROR]",  # Show status indicator
+        "⚠️": "[WARNING]",  # Show status indicator
+        "⏱️": "",  # Just omit for timing
+    }
+
+    # Apply emoji safety to the format string
+    safe_format_string = format_string
+    for emoji, fallback in emoji_map.items():
+        if emoji in format_string:  # Only process if emoji is actually present
+            safe_format_string = safe_format_string.replace(
+                emoji, safe_emoji(emoji, fallback)
+            )
+
+    # Format with the processed string
+    return safe_format_string.format(*args, **kwargs)
+
+
+def safe_print(message: str, **print_kwargs: Any) -> None:
+    """Print message with emoji safety.
+
+    Convenience function for safe printing with automatic emoji handling.
+
+    Args:
+        message: Message to print (may contain emoji)
+        **print_kwargs: Additional arguments passed to print()
+    """
+    # Fast path for messages without emoji
+    emoji_chars = {
+        "🚀",
+        "🔍",
+        "⚙️",
+        "ℹ️",
+        "📖",
+        "💻",
+        "📄",
+        "🌐",
+        "🕐",
+        "📋",
+        "🤖",
+        "🔒",
+        "🛠️",
+        "📎",
+        "📥",
+        "📊",
+        "💰",
+        "✅",
+        "❌",
+        "⚠️",
+        "⏱️",
+    }
+    if not any(emoji in message for emoji in emoji_chars):
+        print(message, **print_kwargs)
+        return
+
+    # Process emoji for compatibility
+    safe_message = safe_format(message)
+    print(safe_message, **print_kwargs)