ostruct-cli 0.8.2__py3-none-any.whl → 0.8.3__py3-none-any.whl

ostruct/cli/click_options.py

@@ -29,6 +29,56 @@ CommandDecorator = Callable[[F], Command]
 DecoratedCommand = Union[Command, Callable[..., Any]]
 
 
+def parse_feature_flags(
+    enabled_features: tuple[str, ...], disabled_features: tuple[str, ...]
+) -> dict[str, str]:
+    """Parse feature flags from CLI arguments.
+
+    Args:
+        enabled_features: Tuple of feature names to enable
+        disabled_features: Tuple of feature names to disable
+
+    Returns:
+        Dictionary mapping feature names to "on" or "off"
+
+    Raises:
+        click.BadParameter: If flag format is invalid or conflicts exist
+    """
+    parsed = {}
+
+    # Process enabled features
+    for feature in enabled_features:
+        feature = feature.strip()
+        if not feature:
+            raise click.BadParameter("Feature name cannot be empty")
+
+        # Validate known feature flags
+        if feature == "ci-download-hack":
+            parsed[feature] = "on"
+        else:
+            raise click.BadParameter(f"Unknown feature: {feature}")
+
+    # Process disabled features
+    for feature in disabled_features:
+        feature = feature.strip()
+        if not feature:
+            raise click.BadParameter("Feature name cannot be empty")
+
+        # Check for conflicts
+        if feature in parsed:
+            raise click.BadParameter(
+                f"Feature '{feature}' cannot be both enabled and disabled"
+            )
+
+        # Validate known feature flags
+        if feature == "ci-download-hack":
+            parsed[feature] = "off"
+        else:
+            raise click.BadParameter(f"Unknown feature: {feature}")
+
+    return parsed
+
+
 def debug_options(f: Union[Command, Callable[..., Any]]) -> Command:
     """Add debug-related CLI options."""
     # Initial conversion to Command if needed
@@ -573,6 +623,31 @@ def code_interpreter_options(f: Union[Command, Callable[..., Any]]) -> Command:
         help="""Clean up uploaded files after execution to save storage quota.""",
     )(cmd)
 
+    # Feature flags for experimental features
+    cmd = click.option(
+        "--enable-feature",
+        "enabled_features",
+        multiple=True,
+        metavar="<FEATURE>",
+        help="""🔧 [EXPERIMENTAL] Enable experimental features.
+        Available features:
+        • ci-download-hack - Enable two-pass sentinel mode for reliable Code Interpreter
+          file downloads with structured output. Overrides config file setting.
+        Example: --enable-feature ci-download-hack""",
+    )(cmd)
+
+    cmd = click.option(
+        "--disable-feature",
+        "disabled_features",
+        multiple=True,
+        metavar="<FEATURE>",
+        help="""🔧 [EXPERIMENTAL] Disable experimental features.
+        Available features:
+        • ci-download-hack - Force single-pass mode for Code Interpreter downloads.
+          Overrides config file setting.
+        Example: --disable-feature ci-download-hack""",
+    )(cmd)
+
     return cast(Command, cmd)
 
 
@@ -685,13 +760,17 @@ def web_search_options(f: Union[Command, Callable[..., Any]]) -> Command:
         is_flag=True,
         help="""🌐 [WEB SEARCH] Enable OpenAI web search tool for up-to-date information.
         Allows the model to search the web for current events, recent updates, and real-time data.
-        Note: Search queries may be sent to external services via OpenAI.""",
+        Note: Search queries may be sent to external services via OpenAI.
+
+        ⚠️  DEPRECATED: Use --enable-tool web-search instead. Will be removed in v0.9.0.""",
     )(cmd)
 
     cmd = click.option(
         "--no-web-search",
         is_flag=True,
-        help="""Explicitly disable web search even if enabled by default in configuration.""",
+        help="""Explicitly disable web search even if enabled by default in configuration.
+
+        ⚠️  DEPRECATED: Use --disable-tool web-search instead. Will be removed in v0.9.0.""",
     )(cmd)
 
     cmd = click.option(
@@ -725,6 +804,35 @@ def web_search_options(f: Union[Command, Callable[..., Any]]) -> Command:
     return cast(Command, cmd)
 
 
+def tool_toggle_options(f: Union[Command, Callable[..., Any]]) -> Command:
+    """Add universal tool toggle CLI options."""
+    cmd: Any = f if isinstance(f, Command) else f
+
+    cmd = click.option(
+        "--enable-tool",
+        "enabled_tools",
+        multiple=True,
+        metavar="<TOOL>",
+        help="""🔧 [TOOL TOGGLES] Enable a tool for this run (repeatable).
+        Overrides configuration file and implicit activation.
+        Available tools: code-interpreter, file-search, web-search, mcp
+        Example: --enable-tool code-interpreter --enable-tool web-search""",
+    )(cmd)
+
+    cmd = click.option(
+        "--disable-tool",
+        "disabled_tools",
+        multiple=True,
+        metavar="<TOOL>",
+        help="""🔧 [TOOL TOGGLES] Disable a tool for this run (repeatable).
+        Overrides configuration file and implicit activation.
+        Available tools: code-interpreter, file-search, web-search, mcp
+        Example: --disable-tool web-search --disable-tool mcp""",
+    )(cmd)
+
+    return cast(Command, cmd)
+
+
 def debug_progress_options(f: Union[Command, Callable[..., Any]]) -> Command:
     """Add debugging and progress CLI options."""
     cmd: Any = f if isinstance(f, Command) else f
@@ -746,12 +854,6 @@ def debug_progress_options(f: Union[Command, Callable[..., Any]]) -> Command:
         "--verbose", is_flag=True, help="Enable verbose logging"
     )(cmd)
 
-    cmd = click.option(
-        "--debug-openai-stream",
-        is_flag=True,
-        help="Debug OpenAI streaming process",
-    )(cmd)
-
     cmd = click.option(
         "--timeout",
         type=int,
@@ -777,6 +879,7 @@ def all_options(f: Union[Command, Callable[..., Any]]) -> Command:
     cmd = code_interpreter_options(cmd)
     cmd = file_search_options(cmd)
     cmd = web_search_options(cmd)
+    cmd = tool_toggle_options(cmd)
     cmd = debug_options(cmd)
     cmd = debug_progress_options(cmd)
 

ostruct/cli/code_interpreter.py

@@ -7,7 +7,7 @@ and integrating code execution capabilities with the OpenAI Responses API.
 import logging
 import os
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from openai import AsyncOpenAI
 
@@ -17,14 +17,18 @@ logger = logging.getLogger(__name__)
 class CodeInterpreterManager:
     """Manager for Code Interpreter file uploads and tool integration."""
 
-    def __init__(self, client: AsyncOpenAI):
+    def __init__(
+        self, client: AsyncOpenAI, config: Optional[Dict[str, Any]] = None
+    ):
         """Initialize Code Interpreter manager.
 
         Args:
             client: AsyncOpenAI client instance
+            config: Code interpreter configuration dict
         """
         self.client = client
         self.uploaded_file_ids: List[str] = []
+        self.config = config or {}
 
     async def upload_files_for_code_interpreter(
         self, files: List[str]
@@ -96,13 +100,75 @@ class CodeInterpreterManager:
             "container": {"type": "auto", "file_ids": file_ids},
         }
 
+    def _collect_file_annotations(self, resp: Any) -> List[Dict[str, Any]]:
+        """Collect file annotations from Responses API output.
+
+        Based on IMPLEMENTATION_NOTES.md findings:
+        - resp.output is a list of ResponseCodeInterpreterToolCall and ResponseOutputMessage
+        - Annotations can be in ResponseOutputMessage.content[].annotations
+        - Also check ResponseCodeInterpreterToolCall for file outputs
+        - Look for container_file_citation type
+
+        Returns:
+            List of annotation dicts with file_id, container_id, filename
+        """
+        annotations = []
+
+        for item in resp.output:
+            # Check messages for annotations
+            if getattr(item, "type", None) == "message":
+                for blk in item.content or []:
+                    if hasattr(blk, "annotations"):
+                        for ann in blk.annotations or []:
+                            # Look specifically for container_file_citation type
+                            if (
+                                getattr(ann, "type", None)
+                                == "container_file_citation"
+                            ):
+                                annotations.append(
+                                    {
+                                        "file_id": ann.file_id,
+                                        "container_id": getattr(
+                                            ann, "container_id", None
+                                        ),
+                                        "filename": getattr(
+                                            ann, "filename", None
+                                        ),
+                                        "type": ann.type,
+                                    }
+                                )
+
+            # Check code interpreter tool calls for file outputs
+            elif getattr(item, "type", None) == "code_interpreter_call":
+                # Check if the tool call has outputs with files
+                if hasattr(item, "outputs"):
+                    for output in item.outputs or []:
+                        # Look for file outputs
+                        if hasattr(output, "type") and output.type == "file":
+                            file_id = getattr(output, "file_id", None)
+                            filename = getattr(output, "filename", None)
+                            if file_id:
+                                annotations.append(
+                                    {
+                                        "file_id": file_id,
+                                        "container_id": None,
+                                        "filename": filename or file_id,
+                                        "type": "code_interpreter_file",
+                                    }
+                                )
+
+        return annotations
+
     async def download_generated_files(
-        self, response_file_ids: List[str], output_dir: str = "."
+        self, response: Any, output_dir: str = "."
     ) -> List[str]:
-        """Download files generated by Code Interpreter.
+        """Download files generated by Code Interpreter using annotations.
+
+        Updated to use container_file_citation annotations instead of
+        deprecated message.file_ids field.
 
         Args:
-            response_file_ids: List of file IDs from Code Interpreter response
+            response: Response from client.responses.create()
             output_dir: Directory to save downloaded files
 
         Returns:
@@ -111,35 +177,162 @@ class CodeInterpreterManager:
         Raises:
             Exception: If download fails
         """
-        downloaded_paths = []
+        if not response:
+            return []
+
+        # Check if auto_download is enabled
+        if not self.config.get("auto_download", True):
+            logger.debug("Auto-download disabled in configuration")
+            return []
+
+        # Ensure output directory exists
         output_path = Path(output_dir)
         output_path.mkdir(exist_ok=True)
 
-        for file_id in response_file_ids:
+        # Collect file annotations using new method
+        annotations = self._collect_file_annotations(response)
+        logger.debug(
+            f"Found {len(annotations)} file annotations: {annotations}"
+        )
+
+        if not annotations:
+            logger.debug("No file annotations found in response")
+            return []
+
+        downloaded_paths = []
+
+        for ann in annotations:
             try:
-                # Get file info
-                file_info = await self.client.files.retrieve(file_id)
-                filename = (
-                    file_info.filename or f"generated_file_{file_id[:8]}.dat"
-                )
+                file_id = ann["file_id"]
+                container_id = ann.get("container_id")
+                filename = ann.get("filename") or file_id
 
-                # Download file content
-                file_content = await self.client.files.content(file_id)
+                # Use container-specific API for cfile_* IDs
+                if file_id.startswith("cfile_") and container_id:
+                    logger.debug(
+                        f"Using container API for {file_id} in container {container_id}"
+                    )
+
+                    # Try different approaches to access the Container Files API
+                    file_content = None
+
+                    # Approach 1: Direct method call (should work in v1.84.0+)
+                    try:
+                        logger.debug(
+                            "Attempting direct containers.files.content() call"
+                        )
+                        # Note: type: ignore[operator] is needed due to mypy false positive
+                        # The OpenAI SDK's type annotations incorrectly suggest AsyncContent is not callable
+                        # but the method works correctly at runtime and returns an object with .content property
+                        result = await self.client.containers.files.content(  # type: ignore[operator]
+                            file_id, container_id=container_id
+                        )
+
+                        # Based on expert guidance: in v1.83.0+, result should have .content property
+                        if hasattr(result, "content"):
+                            file_content = result.content
+                            logger.debug(
+                                f"✓ Got content via result.content: {len(file_content)} bytes"
+                            )
+                        elif hasattr(result, "response") and hasattr(
+                            result.response, "content"
+                        ):
+                            file_content = result.response.content
+                            logger.debug(
+                                f"✓ Got content via result.response.content: {len(file_content)} bytes"
+                            )
+                        else:
+                            logger.debug(
+                                f"Result type: {type(result)}, available attrs: {[a for a in dir(result) if not a.startswith('_')]}"
+                            )
+
+                    except Exception as e:
+                        logger.debug(f"Direct method call failed: {e}")
+
+                    # Approach 2: Try using the raw HTTP client if direct method fails
+                    if file_content is None:
+                        try:
+                            logger.debug(
+                                "Attempting raw HTTP request to container files endpoint"
+                            )
+                            import httpx
+
+                            # Construct the URL manually
+                            base_url = str(self.client.base_url).rstrip("/")
+                            url = f"{base_url}/containers/{container_id}/files/{file_id}/content"
+
+                            # Use the client's HTTP client with auth headers
+                            headers = {
+                                "Authorization": f"Bearer {self.client.api_key}",
+                                "User-Agent": "ostruct/container-files-client",
+                            }
+
+                            async with httpx.AsyncClient() as http_client:
+                                response = await http_client.get(
+                                    url, headers=headers
+                                )
+                                response.raise_for_status()
+                                file_content = response.content
+                                logger.debug(
+                                    f"✓ Got content via raw HTTP: {len(file_content)} bytes"
+                                )
+
+                        except Exception as e:
+                            logger.debug(f"Raw HTTP request failed: {e}")
+
+                    if file_content is None:
+                        raise Exception(
+                            f"Failed to download container file {file_id} using both direct API and raw HTTP methods"
+                        )
+                else:
+                    logger.debug(f"Using standard Files API for {file_id}")
+                    # Use standard Files API for regular uploaded files
+                    file_content_resp = await self.client.files.content(
+                        file_id
+                    )
+                    file_content = file_content_resp.read()
 
                 # Save to local file
                 local_path = output_path / filename
                 with open(local_path, "wb") as f:
-                    f.write(file_content.read())
+                    f.write(file_content)
 
                 downloaded_paths.append(str(local_path))
-                logger.debug(f"Downloaded generated file: {local_path}")
+                logger.info(f"Downloaded generated file: {local_path}")
 
             except Exception as e:
                 logger.error(f"Failed to download file {file_id}: {e}")
-                raise
+                # Continue with other files instead of raising
+                continue
 
         return downloaded_paths
 
+    def _extract_filename_from_message(self, msg: Any) -> str:
+        """Extract filename from message content if available.
+
+        Args:
+            msg: Message object that might contain filename references
+
+        Returns:
+            Extracted filename or empty string if not found
+        """
+        try:
+            # Try to extract filename from markdown links in message content
+            if hasattr(msg, "content") and msg.content:
+                import re
+
+                # Look for patterns like [filename.ext](sandbox:/mnt/data/filename.ext)
+                content_str = str(msg.content)
+                match = re.search(
+                    r"\[([^\]]+\.[a-zA-Z0-9]+)\]\(sandbox:/mnt/data/[^)]+\)",
+                    content_str,
+                )
+                if match:
+                    return match.group(1)
+        except Exception:
+            pass
+        return ""
+
     async def cleanup_uploaded_files(self) -> None:
         """Clean up uploaded files from OpenAI storage.
 

ostruct/cli/commands/run.py

@@ -24,6 +24,28 @@ from ..types import CLIParams
 logger = logging.getLogger(__name__)
 
 
+def _emit_deprecation_warnings(params: CLIParams) -> None:
+    """Emit deprecation warnings for legacy tool-specific flags."""
+    import warnings
+
+    # Web Search flags
+    if params.get("web_search"):
+        warnings.warn(
+            "The --web-search flag is deprecated and will be removed in v0.9.0. "
+            "Use --enable-tool web-search instead.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
+    if params.get("no_web_search"):
+        warnings.warn(
+            "The --no-web-search flag is deprecated and will be removed in v0.9.0. "
+            "Use --disable-tool web-search instead.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
+
 @click.command()
 @click.argument("task_template", type=click.Path(exists=True))
 @click.argument("schema_file", type=click.Path(exists=True))
@@ -91,6 +113,40 @@ def run(
         for k, v in kwargs.items():
             params[k] = v  # type: ignore[literal-required]
 
+        # Process tool toggle flags (Step 2: Conflict guard & normalisation)
+        from typing import Tuple
+
+        enabled_tools_raw: Tuple[str, ...] = params.get("enabled_tools", ())  # type: ignore[assignment]
+        disabled_tools_raw: Tuple[str, ...] = params.get("disabled_tools", ())  # type: ignore[assignment]
+
+        logger.debug(f"Raw enabled tools: {enabled_tools_raw}")
+        logger.debug(f"Raw disabled tools: {disabled_tools_raw}")
+
+        # Ensure we have lists to iterate over (Click returns tuples for multiple=True)
+        enabled_list: list[str] = list(enabled_tools_raw)
+        disabled_list: list[str] = list(disabled_tools_raw)
+
+        enabled_tools = {t.lower() for t in enabled_list}
+        disabled_tools = {t.lower() for t in disabled_list}
+
+        logger.debug(f"Enabled tools normalized: {enabled_tools}")
+        logger.debug(f"Disabled tools normalized: {disabled_tools}")
+
+        # Check for conflicts
+        dupes = enabled_tools & disabled_tools
+        if dupes:
+            logger.error(f"Tool conflict detected: {dupes}")
+            raise click.UsageError(
+                f"--enable-tool and --disable-tool both specified for: {', '.join(sorted(dupes))}"
+            )
+
+        # Store normalized tool toggles for later stages
+        params["_enabled_tools"] = enabled_tools  # type: ignore[typeddict-unknown-key]
+        params["_disabled_tools"] = disabled_tools  # type: ignore[typeddict-unknown-key]
+
+        # Emit deprecation warnings for legacy tool-specific flags
+        _emit_deprecation_warnings(params)
+
         # Apply configuration defaults if values not explicitly provided
         # Check for command-level config option first, then group-level
         command_config = kwargs.get("config")

ostruct/cli/config.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from typing import Any, Dict, Optional, Union
 
 import yaml
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, field_validator, model_validator
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +43,7 @@ class ToolsConfig(BaseModel):
         default_factory=lambda: {
             "auto_download": True,
             "output_directory": "./output",
+            "download_strategy": "single_pass",  # "single_pass" | "two_pass_sentinel"
         }
     )
     file_search: Dict[str, Any] = Field(
@@ -91,6 +92,24 @@ class OstructConfig(BaseModel):
     operation: OperationConfig = Field(default_factory=OperationConfig)
     limits: LimitsConfig = Field(default_factory=LimitsConfig)
 
+    @model_validator(mode="before")
+    @classmethod
+    def _validate_download_strategy(cls, values: Any) -> Any:
+        """Validate download_strategy in code_interpreter config."""
+        if isinstance(values, dict):
+            tools_config = values.get("tools", {})
+            if isinstance(tools_config, dict):
+                ci_config = tools_config.get("code_interpreter", {})
+                if isinstance(ci_config, dict):
+                    strategy = ci_config.get(
+                        "download_strategy", "single_pass"
+                    )
+                    if strategy not in {"single_pass", "two_pass_sentinel"}:
+                        raise ValueError(
+                            "download_strategy must be 'single_pass' or 'two_pass_sentinel'"
+                        )
+        return values
+
     @classmethod
     def load(
         cls, config_path: Optional[Union[str, Path]] = None

ostruct/cli/errors.py

@@ -392,24 +392,6 @@ class ModelNotSupportedError(CLIError):
     pass
 
 
-class StreamInterruptedError(CLIError):
-    """Exception raised when a stream is interrupted."""
-
-    pass
-
-
-class StreamBufferError(CLIError):
-    """Exception raised when there's an error with the stream buffer."""
-
-    pass
-
-
-class StreamParseError(CLIError):
-    """Exception raised when there's an error parsing the stream."""
-
-    pass
-
-
 class APIResponseError(CLIError):
     """Exception raised when there's an error with the API response."""
 
@@ -788,15 +770,8 @@ def handle_error(e: Exception) -> None:
             logger.debug(
                 f"Error details:\nType: {type(e).__name__}\n{context_str.rstrip()}"
             )
-    elif not isinstance(
-        e,
-        (
-            click.UsageError,
-            DuplicateFileMappingError,
-            VariableNameError,
-            VariableValueError,
-        ),
-    ):
+    elif not isinstance(e, (CLIError, click.UsageError)):
+        # Only show tracebacks for truly unexpected errors (not CLIError subclasses)
         logger.error(msg, exc_info=True)
 
     # 3. User output
@@ -820,9 +795,6 @@ __all__ = [
     "InvalidJSONError",
     "ModelCreationError",
     "ModelNotSupportedError",
-    "StreamInterruptedError",
-    "StreamBufferError",
-    "StreamParseError",
     "APIResponseError",
     "EmptyResponseError",
     "InvalidResponseFormatError",