ostruct-cli 0.8.29__py3-none-any.whl → 1.0.1__py3-none-any.whl

ostruct/cli/code_interpreter.py

@@ -7,10 +7,20 @@ and integrating code execution capabilities with the OpenAI Responses API.
 import logging
 import os
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from openai import AsyncOpenAI
 
+from .errors import (
+    DownloadError,
+    DownloadFileNotFoundError,
+    DownloadNetworkError,
+    DownloadPermissionError,
+)
+
+if TYPE_CHECKING:
+    from .upload_manager import SharedUploadManager
+
 logger = logging.getLogger(__name__)
 
 
@@ -18,17 +28,22 @@ class CodeInterpreterManager:
     """Manager for Code Interpreter file uploads and tool integration."""
 
     def __init__(
-        self, client: AsyncOpenAI, config: Optional[Dict[str, Any]] = None
-    ):
+        self,
+        client: AsyncOpenAI,
+        config: Optional[Dict[str, Any]] = None,
+        upload_manager: Optional["SharedUploadManager"] = None,
+    ) -> None:
         """Initialize Code Interpreter manager.
 
         Args:
             client: AsyncOpenAI client instance
             config: Code interpreter configuration dict
+            upload_manager: Optional shared upload manager for deduplication
         """
         self.client = client
         self.uploaded_file_ids: List[str] = []
         self.config = config or {}
+        self.upload_manager = upload_manager
 
     async def upload_files_for_code_interpreter(
         self, files: List[str]
@@ -83,6 +98,32 @@ class CodeInterpreterManager:
         self.uploaded_file_ids.extend(file_ids)
         return file_ids
 
+    async def get_files_from_shared_manager(self) -> List[str]:
+        """Get file IDs from shared upload manager for Code Interpreter.
+
+        Returns:
+            List of OpenAI file IDs for Code Interpreter use
+        """
+        if not self.upload_manager:
+            logger.warning("No shared upload manager available")
+            return []
+
+        # Trigger upload processing for code-interpreter tool
+        await self.upload_manager.upload_for_tool("code-interpreter")
+
+        # Get the uploaded file IDs
+        file_ids = list(
+            self.upload_manager.get_files_for_tool("code-interpreter")
+        )
+
+        # Track for cleanup
+        self.uploaded_file_ids.extend(file_ids)
+
+        logger.debug(
+            f"Retrieved {len(file_ids)} file IDs from shared manager for CI"
+        )
+        return file_ids
+
     def build_tool_config(self, file_ids: List[str]) -> dict:
         """Build Code Interpreter tool configuration.
 
@@ -187,7 +228,14 @@ class CodeInterpreterManager:
 
         # Ensure output directory exists
         output_path = Path(output_dir)
-        output_path.mkdir(exist_ok=True)
+        try:
+            output_path.mkdir(exist_ok=True)
+        except PermissionError as e:
+            raise DownloadPermissionError(str(output_path)) from e
+        except OSError as e:
+            raise DownloadError(
+                f"Failed to create download directory: {e}"
+            ) from e
 
         # Collect file annotations using new method
         annotations = self._collect_file_annotations(response)
@@ -292,21 +340,156 @@ class CodeInterpreterManager:
                     )
                     file_content = file_content_resp.read()
 
-                # Save to local file
+                # Handle file naming conflicts
                 local_path = output_path / filename
-                with open(local_path, "wb") as f:
-                    f.write(file_content)
+                resolved_path = self._handle_file_conflict(local_path)
 
-                downloaded_paths.append(str(local_path))
-                logger.info(f"Downloaded generated file: {local_path}")
+                if resolved_path is None:
+                    # Skip this file according to conflict resolution strategy
+                    logger.info(f"Skipping existing file: {local_path}")
+                    continue
 
+                # Save to resolved local file path
+                try:
+                    with open(resolved_path, "wb") as f:
+                        f.write(file_content)
+                except PermissionError as e:
+                    raise DownloadPermissionError(
+                        str(resolved_path.parent)
+                    ) from e
+                except OSError as e:
+                    raise DownloadError(
+                        f"Failed to write file {resolved_path}: {e}"
+                    ) from e
+
+                # Validate the downloaded file
+                self._validate_downloaded_file(resolved_path)
+
+                downloaded_paths.append(str(resolved_path))
+                logger.info(f"Downloaded generated file: {resolved_path}")
+
+            except DownloadError:
+                # Re-raise download-specific errors without modification
+                raise
+            except FileNotFoundError as e:
+                raise DownloadFileNotFoundError(file_id) from e
             except Exception as e:
-                logger.error(f"Failed to download file {file_id}: {e}")
-                # Continue with other files instead of raising
-                continue
+                # Check if it's a network-related error
+                if any(
+                    keyword in str(e).lower()
+                    for keyword in ["network", "connection", "timeout", "http"]
+                ):
+                    raise DownloadNetworkError(
+                        file_id, original_error=e
+                    ) from e
+                else:
+                    logger.error(f"Failed to download file {file_id}: {e}")
+                    # Continue with other files instead of raising
+                    continue
 
         return downloaded_paths
 
+    def _handle_file_conflict(self, local_path: Path) -> Optional[Path]:
+        """Handle file naming conflicts based on configuration.
+
+        Args:
+            local_path: The original path where file would be saved
+
+        Returns:
+            Resolved path where file should be saved, or None to skip
+        """
+        if not local_path.exists():
+            return local_path
+
+        strategy = self.config.get("duplicate_outputs", "overwrite")
+
+        if strategy == "overwrite":
+            logger.info(f"Overwriting existing file: {local_path}")
+            return local_path
+
+        elif strategy == "rename":
+            # Generate unique name: file.txt -> file_1.txt, file_2.txt, etc.
+            counter = 1
+            stem = local_path.stem
+            suffix = local_path.suffix
+            parent = local_path.parent
+
+            while True:
+                new_path = parent / f"{stem}_{counter}{suffix}"
+                if not new_path.exists():
+                    logger.info(f"File exists, using: {new_path}")
+                    return new_path
+                counter += 1
+
+        elif strategy == "skip":
+            logger.info(f"File exists, skipping: {local_path}")
+            return None  # Signal to skip this file
+
+        # Default to overwrite for unknown strategies
+        logger.warning(
+            f"Unknown duplicate_outputs strategy '{strategy}', defaulting to overwrite"
+        )
+        return local_path
+
+    def _validate_downloaded_file(self, file_path: Path) -> None:
+        """Perform basic validation of downloaded files.
+
+        Args:
+            file_path: Path to the downloaded file
+
+        Note:
+            This only logs warnings, it does not block downloads.
+        """
+        validation_level = self.config.get("output_validation", "basic")
+
+        if validation_level == "off":
+            return
+
+        try:
+            # Check file size (prevent huge files)
+            size = file_path.stat().st_size
+            size_mb = size / (1024 * 1024)
+
+            if size_mb > 100:  # 100MB limit
+                logger.warning(
+                    f"Large file downloaded: {file_path} ({size_mb:.1f}MB)"
+                )
+
+            # Check for potentially dangerous file types
+            dangerous_extensions = {
+                ".exe",
+                ".bat",
+                ".sh",
+                ".cmd",
+                ".com",
+                ".scr",
+                ".vbs",
+                ".js",
+            }
+            if file_path.suffix.lower() in dangerous_extensions:
+                logger.warning(
+                    f"Potentially dangerous file type downloaded: {file_path} "
+                    f"(extension: {file_path.suffix})"
+                )
+
+            # In strict mode, perform additional checks
+            if validation_level == "strict":
+                # Check for hidden files
+                if file_path.name.startswith("."):
+                    logger.warning(f"Hidden file downloaded: {file_path}")
+
+                # Check for files with multiple extensions (potential masquerading)
+                parts = file_path.name.split(".")
+                if len(parts) > 2:
+                    logger.warning(
+                        f"File with multiple extensions downloaded: {file_path} "
+                        f"(could be masquerading)"
+                    )
+
+        except Exception as e:
+            logger.debug(f"Error during file validation: {e}")
+            # Don't fail downloads due to validation errors
+
     def _extract_filename_from_message(self, msg: Any) -> str:
         """Extract filename from message content if available.
 

ostruct/cli/commands/__init__.py

@@ -3,7 +3,6 @@
 import click
 
 from .list_models import list_models
-from .quick_ref import quick_reference
 from .run import run
 from .update_registry import update_registry
 
@@ -15,7 +14,6 @@ def create_command_group() -> click.Group:
 
     # Add all commands to the group
     group.add_command(run)
-    group.add_command(quick_reference)
     group.add_command(update_registry)
     group.add_command(list_models)
 
@@ -25,7 +23,6 @@ def create_command_group() -> click.Group:
 # Export commands for easy importing
 __all__ = [
     "run",
-    "quick_reference",
     "update_registry",
     "list_models",
     "create_command_group",

ostruct/cli/commands/run.py

@@ -4,9 +4,10 @@ import asyncio
 import json
 import logging
 import sys
-from typing import Any
+from pathlib import Path
+from typing import Any, Tuple
 
-import click
+import rich_click as click
 
 from ..click_options import all_options
 from ..config import OstructConfig
@@ -24,29 +25,12 @@ from ..types import CLIParams
 logger = logging.getLogger(__name__)
 
 
-def _emit_deprecation_warnings(params: CLIParams) -> None:
-    """Emit deprecation warnings for legacy tool-specific flags."""
-    import warnings
-
-    # Web Search flags
-    if params.get("web_search"):
-        warnings.warn(
-            "The --web-search flag is deprecated and will be removed in v0.9.0. "
-            "Use --enable-tool web-search instead.",
-            DeprecationWarning,
-            stacklevel=3,
-        )
-
-    if params.get("no_web_search"):
-        warnings.warn(
-            "The --no-web-search flag is deprecated and will be removed in v0.9.0. "
-            "Use --disable-tool web-search instead.",
-            DeprecationWarning,
-            stacklevel=3,
-        )
-
-
-@click.command()
+@click.command(
+    cls=click.RichCommand,
+    context_settings={
+        "help_option_names": ["-h", "--help"],
+    },
+)
 @click.argument("task_template", type=click.Path(exists=True))
 @click.argument("schema_file", type=click.Path(exists=True))
 @all_options
@@ -57,50 +41,46 @@ def run(
     schema_file: str,
     **kwargs: Any,
 ) -> None:
-    """Run structured output generation with multi-tool integration.
+    """Transform unstructured inputs into structured JSON using OpenAI APIs, Jinja2 templates, and powerful tool integrations.
+
+    🚀 QUICK START
 
-    \b
-    📁 FILE ROUTING OPTIONS:
+    ostruct run template.j2 schema.json -V name=value
 
-    Template Access Only:
-      -ft, --file-for-template FILE     Files available in template only
-      -dt, --dir-for-template DIR       Directories for template access
+    📎 FILE ATTACHMENT
 
-    Code Interpreter (execution & analysis):
-      -fc, --file-for-code-interpreter FILE    Upload files for code execution
-      -dc, --dir-for-code-interpreter DIR      Upload directories for analysis
+    --file data file.txt              Template access (default)
 
-    File Search (document retrieval):
-      -fs, --file-for-file-search FILE         Upload files for vector search
-      -ds, --dir-for-search DIR                Upload directories for search
+    --file ci:data data.csv           Code Interpreter upload
 
-    Advanced Routing:
-      --file-for TOOL PATH              Route files to specific tools
-                                        Example: --file-for code-interpreter data.json
+    --file fs:docs manual.pdf         File Search upload
 
-    \b
-    🔧 TOOL INTEGRATION:
+    🔧 TOOL INTEGRATION
 
-    MCP Servers:
-      --mcp-server [LABEL@]URL          Connect to MCP server
-                                        Example: --mcp-server deepwiki@https://mcp.deepwiki.com/sse
+    --enable-tool code-interpreter    Code execution & analysis
 
-    \b
-    ⚡ EXAMPLES:
+    --enable-tool file-search         Document search & retrieval
 
-    Basic usage:
-      ostruct run template.j2 schema.json -V name=value
+    --enable-tool web-search          Real-time web information
 
-    Multi-tool explicit routing:
-      ostruct run analysis.j2 schema.json -fc data.csv -fs docs.pdf -ft config.yaml
+    🔧 ENVIRONMENT VARIABLES
 
-    Legacy compatibility (still works):
-      ostruct run template.j2 schema.json -f config main.py -d src ./src
+    ```text
+    Core API Configuration:
+    OPENAI_API_KEY                           OpenAI API authentication key
+    OPENAI_API_BASE                          Custom OpenAI API base URL
 
-    \b
-    Arguments:
-      TASK_TEMPLATE  Path to Jinja2 template file
-      SCHEMA_FILE    Path to JSON schema file defining output structure
+    Template Processing Limits:
+    OSTRUCT_TEMPLATE_FILE_LIMIT              Max individual file size (default: 64KB)
+    OSTRUCT_TEMPLATE_TOTAL_LIMIT             Max total files size (default: 1MB)
+    OSTRUCT_TEMPLATE_PREVIEW_LIMIT           Template preview size limit (default: 4096)
+
+    System Behavior:
+    OSTRUCT_DISABLE_REGISTRY_UPDATE_CHECKS   Disable model registry updates
+    OSTRUCT_MCP_URL_<name>                   Custom MCP server URLs
+    ```
+
+    See organized option groups below for complete functionality.
     """
     try:
         # Convert Click parameters to typed dict
@@ -113,9 +93,19 @@ def run(
         for k, v in kwargs.items():
             params[k] = v  # type: ignore[literal-required]
 
-        # Process tool toggle flags (Step 2: Conflict guard & normalisation)
-        from typing import Tuple
+        # UNIFIED GUIDELINES: Validate JSON flag combinations
+        if kwargs.get("dry_run_json") and not kwargs.get("dry_run"):
+            raise click.BadOptionUsage(
+                "--dry-run-json", "--dry-run-json requires --dry-run"
+            )
 
+        if kwargs.get("run_summary_json") and kwargs.get("dry_run"):
+            raise click.BadOptionUsage(
+                "--run-summary-json",
+                "--run-summary-json cannot be used with --dry-run",
+            )
+
+        # Process tool toggle flags (Step 2: Conflict guard & normalisation)
         enabled_tools_raw: Tuple[str, ...] = params.get("enabled_tools", ())  # type: ignore[assignment]
         disabled_tools_raw: Tuple[str, ...] = params.get("disabled_tools", ())  # type: ignore[assignment]
 
@@ -144,9 +134,6 @@ def run(
         params["_enabled_tools"] = enabled_tools  # type: ignore[typeddict-unknown-key]
         params["_disabled_tools"] = disabled_tools  # type: ignore[typeddict-unknown-key]
 
-        # Emit deprecation warnings for legacy tool-specific flags
-        _emit_deprecation_warnings(params)
-
         # Apply configuration defaults if values not explicitly provided
         # Check for command-level config option first, then group-level
         command_config = kwargs.get("config")
@@ -158,6 +145,246 @@ def run(
         if params.get("model") is None:
             params["model"] = config.get_model_default()
 
+        # UNIFIED GUIDELINES: Perform basic validation even in dry-run mode
+        if kwargs.get("dry_run"):
+            # Import validation functions
+            from ..attachment_processor import (
+                AttachmentSpec,
+                ProcessedAttachments,
+            )
+            from ..plan_assembly import PlanAssembler
+            from ..plan_printing import PlanPrinter
+            from ..validators import validate_inputs
+
+            # Variables to track validation state
+            validation_passed = True
+            template_warning = None
+            original_template_path = task_template
+
+            # Process attachments for the dry-run plan
+            processed_attachments = ProcessedAttachments()
+
+            # Process --file attachments
+            files = kwargs.get("attaches", [])
+            for file_spec in files:
+                spec = AttachmentSpec(
+                    alias=file_spec["alias"],
+                    path=file_spec["path"],
+                    targets=file_spec["targets"],
+                    recursive=file_spec.get("recursive", False),
+                    pattern=file_spec.get("pattern"),
+                )
+                processed_attachments.alias_map[spec.alias] = spec
+
+                # Route to appropriate lists based on targets
+                if "prompt" in spec.targets:
+                    processed_attachments.template_files.append(spec)
+                if "code-interpreter" in spec.targets or "ci" in spec.targets:
+                    processed_attachments.ci_files.append(spec)
+                if "file-search" in spec.targets or "fs" in spec.targets:
+                    processed_attachments.fs_files.append(spec)
+
+            # Process --dir attachments
+            dirs = kwargs.get("dirs", [])
+            for dir_spec in dirs:
+                spec = AttachmentSpec(
+                    alias=dir_spec["alias"],
+                    path=dir_spec["path"],
+                    targets=dir_spec["targets"],
+                    recursive=dir_spec.get("recursive", True),
+                    pattern=dir_spec.get("pattern"),
+                )
+                processed_attachments.alias_map[spec.alias] = spec
+
+                # Route to appropriate lists based on targets
+                if "prompt" in spec.targets:
+                    processed_attachments.template_dirs.append(spec)
+                if "code-interpreter" in spec.targets or "ci" in spec.targets:
+                    processed_attachments.ci_dirs.append(spec)
+                if "file-search" in spec.targets or "fs" in spec.targets:
+                    processed_attachments.fs_dirs.append(spec)
+
+            try:
+                # Perform the same input validation as live runs (async)
+                logger.debug("Performing dry-run validation")
+
+                # Run async validation
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                try:
+                    # Use the same params structure as the live run
+                    validation_result = loop.run_until_complete(
+                        validate_inputs(params)
+                    )
+                    # Extract components from the tuple
+                    (
+                        security_manager,
+                        validated_template,
+                        schema,
+                        template_context,
+                        env,
+                        template_path,
+                    ) = validation_result
+
+                    # Update task_template with validated content
+                    task_template = validated_template
+
+                    # Perform template rendering validation to catch binary file access errors
+                    logger.debug("Performing template rendering validation")
+                    from ..template_processor import process_templates
+
+                    system_prompt, user_prompt = loop.run_until_complete(
+                        process_templates(
+                            params,
+                            task_template,
+                            template_context,
+                            env,
+                            template_path,
+                        )
+                    )
+                    logger.debug("Template rendering validation passed")
+
+                    # Check for template warnings by processing system prompt
+                    from typing import cast
+
+                    from ..template_processor import process_system_prompt
+
+                    result = cast(
+                        Tuple[str, bool],
+                        process_system_prompt(
+                            task_template,
+                            params.get("system_prompt"),
+                            params.get("system_prompt_file"),
+                            template_context,
+                            env,
+                            params.get("ignore_task_sysprompt", False),
+                            template_path,
+                        ),
+                    )
+                    _system_prompt_check: str
+                    template_has_conflict: bool
+                    _system_prompt_check, template_has_conflict = result
+
+                    if template_has_conflict:
+                        template_warning = (
+                            "Template has YAML frontmatter with 'system_prompt' field, but --sys-file was also provided. "
+                            "Using --sys-file and ignoring YAML frontmatter system_prompt."
+                        )
+
+                finally:
+                    loop.close()
+
+            except Exception as e:
+                validation_passed = False
+                template_warning = str(e)
+                logger.error(f"Dry-run validation failed: {e}")
+
+                # For critical errors, exit immediately with proper error handling
+                if not isinstance(e, (ValueError, FileNotFoundError)):
+                    handle_error(e)
+                    if hasattr(e, "exit_code"):
+                        ctx.exit(int(e.exit_code))
+                    else:
+                        ctx.exit(1)
+
+            # Build plan with warning information
+            plan_kwargs = {
+                "allowed_paths": params.get("allowed_paths", []),
+                "cost_estimate": None,  # We don't have cost estimate in dry run
+                "template_warning": template_warning,
+                "original_template_path": original_template_path,
+                "validation_passed": validation_passed,
+            }
+
+            # Add enabled tools from routing result and explicit tool toggles
+            plan_enabled_tools: set[str] = set()
+
+            # Get tools from routing result (auto-enabled by file attachments)
+            routing_result = params.get("_routing_result")
+            if routing_result and hasattr(routing_result, "enabled_tools"):
+                plan_enabled_tools.update(routing_result.enabled_tools)
+
+            # Add explicitly enabled tools
+            explicit_enabled = params.get("_enabled_tools", set())
+            if isinstance(explicit_enabled, set):
+                plan_enabled_tools.update(explicit_enabled)
+
+            # Remove explicitly disabled tools
+            explicit_disabled = params.get("_disabled_tools", set())
+            if isinstance(explicit_disabled, set):
+                plan_enabled_tools -= explicit_disabled
+
+            if plan_enabled_tools:
+                plan_kwargs["enabled_tools"] = plan_enabled_tools
+
+            # Add CI configuration for download validation
+            if "code-interpreter" in plan_enabled_tools:
+                config_path = params.get("config")
+                config = OstructConfig.load(
+                    config_path
+                    if isinstance(config_path, (str, Path))
+                    else None
+                )
+                ci_config = config.get_code_interpreter_config()
+                plan_kwargs["ci_config"] = ci_config
+
+            plan = PlanAssembler.build_execution_plan(
+                processed_attachments=processed_attachments,
+                template_path=original_template_path,  # Use original path, not template content
+                schema_path=schema_file,
+                variables=ctx.obj.get("vars", {}) if ctx.obj else {},
+                security_mode=kwargs.get("path_security", "permissive"),
+                model=params.get("model", "gpt-4o"),
+                **plan_kwargs,
+            )
+
+            if kwargs.get("dry_run_json"):
+                # Output JSON to stdout
+                click.echo(json.dumps(plan, indent=2))
+            else:
+                # Output human-readable to stdout
+                PlanPrinter.human(plan)
+
+                # Add debug output for tool states if debug mode is enabled
+                if kwargs.get("debug"):
+                    click.echo("\n--- DEBUG TOOL STATES ---")
+                    debug_enabled_tools = cast(
+                        set[str], params.get("_enabled_tools", set())
+                    )
+                    debug_disabled_tools = cast(
+                        set[str], params.get("_disabled_tools", set())
+                    )
+
+                    # Check web search state
+                    web_search_enabled = "web-search" in debug_enabled_tools
+                    if "web-search" in debug_disabled_tools:
+                        web_search_enabled = False
+                    click.echo(
+                        f"web_search_enabled: bool ({web_search_enabled})"
+                    )
+
+                    # Check code interpreter state
+                    ci_enabled = "code-interpreter" in debug_enabled_tools
+                    if "code-interpreter" in debug_disabled_tools:
+                        ci_enabled = False
+                    # Also enable if CI attachments present
+                    if plan.get("tools", {}).get("code_interpreter", False):
+                        ci_enabled = True
+                    click.echo(
+                        f"code_interpreter_enabled: bool ({ci_enabled})"
+                    )
+
+                # Add completion message based on validation result
+                if validation_passed:
+                    click.echo("\nDry run completed successfully")
+                else:
+                    click.echo(
+                        "\nDry run completed with warnings - see template status above"
+                    )
+
+            # Exit with appropriate code
+            ctx.exit(0 if validation_passed else 1)
+
         # Run the async function synchronously
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)