ostruct-cli 0.8.2__py3-none-any.whl → 0.8.3__py3-none-any.whl

ostruct/cli/sentinel.py

@@ -0,0 +1,29 @@
+"""Sentinel JSON extraction utility for two-pass Code Interpreter workaround."""
+
+import json
+import re
+from typing import Any, Dict, Optional
+
+_SENT_RE = re.compile(r"===BEGIN_JSON===\s*(\{.*?})\s*===END_JSON===", re.S)
+
+
+def extract_json_block(text: str) -> Optional[Dict[str, Any]]:
+    """Extract JSON block from sentinel markers in text.
+
+    Args:
+        text: Response text that may contain sentinel-wrapped JSON
+
+    Returns:
+        Parsed JSON dict if found, None otherwise
+    """
+    m = _SENT_RE.search(text)
+    if not m:
+        return None
+    try:
+        result = json.loads(m.group(1))
+        # Ensure we return a dict, not other JSON types
+        if isinstance(result, dict):
+            return result
+        return None
+    except json.JSONDecodeError:
+        return None

ostruct/cli/template_optimizer.py

@@ -251,19 +251,15 @@ class TemplateOptimizer:
         if match:
             return match.group(1)
 
-        # Pattern 2: variable.content where variable is likely a file
+        # Pattern 2: variable.content - handle ALL single-file variables
         match = re.search(r"(\w+)\.content", content_expr)
         if match:
             var_name = match.group(1)
             # Skip if this is a loop variable
             if var_name in loop_variables:
                 return None
-            # Common file variable patterns
-            if any(
-                keyword in var_name.lower()
-                for keyword in ["file", "config", "data", "script", "code"]
-            ):
-                return f"${var_name}"  # Use variable name as placeholder
+            # Return placeholder for ANY .content access to prevent directory misclassification
+            return f"${var_name}"
 
         # Pattern 3: files['filename'].content
         match = re.search(
@@ -374,6 +370,14 @@ class TemplateOptimizer:
         if dir_var in loop_variables:
             return full_match
 
+        # Skip if this is a single-file variable accessed via .content
+        # These should be handled by the file pattern, not directory pattern
+        if content_expr.endswith(".content"):
+            # Check if this would be handled by _extract_file_path
+            file_path = self._extract_file_path(content_expr, loop_variables)
+            if file_path:
+                return full_match  # Let file pattern handle it
+
         reference = f"the files and subdirectories in <dir:{dir_var}>"
         self.dir_references[dir_var] = reference
         self.optimization_log.append(

ostruct/cli/template_processor.py

@@ -23,6 +23,7 @@ from .errors import (
     VariableNameError,
 )
 from .explicit_file_processor import ProcessingResult
+from .file_info import FileRoutingIntent
 from .file_utils import FileInfoList, collect_files
 from .path_utils import validate_path_mapping
 from .security import SecurityManager
@@ -263,6 +264,31 @@ def process_system_prompt(
         )
         base_prompt += web_search_instructions
 
+    # Add Code Interpreter download instructions if CI is enabled and auto_download is true
+    code_interpreter_enabled = template_context.get(
+        "code_interpreter_enabled", False
+    )
+    auto_download_enabled = template_context.get("auto_download_enabled", True)
+
+    if code_interpreter_enabled and auto_download_enabled:
+        ci_download_instructions = (
+            "\n\nWhen using Code Interpreter to create files, always include markdown download links "
+            "in your response using this exact format: [Download filename.ext](sandbox:/mnt/data/filename.ext). "
+            "This enables automatic file download for the user. Include the download link immediately "
+            "after creating each file."
+        )
+
+        # Add sentinel instructions for two-pass mode
+        ci_config = template_context.get("code_interpreter_config", {})
+        if ci_config.get("download_strategy") == "two_pass_sentinel":
+            ci_download_instructions += (
+                "\n\nAfter saving your files and printing the download links, output "
+                "your JSON response between exactly these markers:\n"
+                "===BEGIN_JSON===\n{ ... }\n===END_JSON===\n"
+            )
+
+        base_prompt += ci_download_instructions
+
     return base_prompt
 
 
@@ -779,25 +805,44 @@ async def create_template_context_from_routing(
             "file-search", []
         )
 
-        # Convert to the format expected by create_template_context
-        # For legacy compatibility, we need (name, path) tuples
-        files_tuples = []
-        seen_files = set()  # Track files to avoid duplicates
+        # Intent mapping is implemented directly in the file categorization logic below
+        # This ensures files from different CLI flags get the correct routing intent
+
+        # Process files by intent groups instead of lumping them all together
+        files_dict = {}
 
-        # Add template files - now single-argument auto-naming only
+        # Group files by their intent
+        files_by_intent: Dict[FileRoutingIntent, List[Tuple[str, str]]] = {
+            FileRoutingIntent.TEMPLATE_ONLY: [],
+            FileRoutingIntent.CODE_INTERPRETER: [],
+            FileRoutingIntent.FILE_SEARCH: [],
+        }
+
+        dirs_by_intent: Dict[FileRoutingIntent, List[Tuple[str, str]]] = {
+            FileRoutingIntent.TEMPLATE_ONLY: [],
+            FileRoutingIntent.CODE_INTERPRETER: [],
+            FileRoutingIntent.FILE_SEARCH: [],
+        }
+
+        # Track processed files to avoid duplicates between CLI args and routing result
+        seen_files: Set[str] = set()
+
+        # Categorize files by their source and assign appropriate intent
+        # Template files from CLI args
         template_file_paths = args.get("template_files", [])
         for template_file_path in template_file_paths:
             if isinstance(template_file_path, (str, Path)):
-                # Auto-generate name for single-arg form: -ft config.yaml
                 file_name = _generate_template_variable_name(
                     str(template_file_path)
                 )
                 file_path_str = str(template_file_path)
                 if file_path_str not in seen_files:
-                    files_tuples.append((file_name, file_path_str))
+                    files_by_intent[FileRoutingIntent.TEMPLATE_ONLY].append(
+                        (file_name, file_path_str)
+                    )
                     seen_files.add(file_path_str)
 
-        # Add template file aliases (from --fta) - two-argument explicit naming
+        # Template file aliases from CLI args
         template_file_aliases = args.get("template_file_aliases", [])
         for name_path_tuple in template_file_aliases:
             if (
@@ -805,51 +850,26 @@ async def create_template_context_from_routing(
                 and len(name_path_tuple) == 2
             ):
                 custom_name, file_path_raw = name_path_tuple
-                file_path = str(file_path_raw)
-                file_name = str(
-                    custom_name
-                )  # Always use custom name for aliases
-
-                if file_path not in seen_files:
-                    files_tuples.append((file_name, file_path))
-                    seen_files.add(file_path)
-
-        # Also process template_files from routing result (for compatibility)
-        for template_file_item in template_files:
-            if isinstance(template_file_item, (str, Path)):
-                file_name = _generate_template_variable_name(
-                    str(template_file_item)
-                )
-                file_path_str = str(template_file_item)
+                file_path_str = str(file_path_raw)
                 if file_path_str not in seen_files:
-                    files_tuples.append((file_name, file_path_str))
+                    files_by_intent[FileRoutingIntent.TEMPLATE_ONLY].append(
+                        (str(custom_name), file_path_str)
+                    )
                     seen_files.add(file_path_str)
-            elif (
-                isinstance(template_file_item, tuple)
-                and len(template_file_item) == 2
-            ):
-                # Handle tuple format (name, path)
-                _, template_file_path = template_file_item
-                template_file_path_str = str(template_file_path)
-                file_name = _generate_template_variable_name(
-                    template_file_path_str
-                )
-                if template_file_path_str not in seen_files:
-                    files_tuples.append((file_name, template_file_path_str))
-                    seen_files.add(template_file_path_str)
 
-        # Add code interpreter files - now single-argument auto-naming only
+        # Code interpreter files from CLI args
         code_interpreter_file_paths = args.get("code_interpreter_files", [])
         for ci_file_path in code_interpreter_file_paths:
             if isinstance(ci_file_path, (str, Path)):
-                # Auto-generate name: -fc data.csv
                 file_name = _generate_template_variable_name(str(ci_file_path))
                 file_path_str = str(ci_file_path)
                 if file_path_str not in seen_files:
-                    files_tuples.append((file_name, file_path_str))
+                    files_by_intent[FileRoutingIntent.CODE_INTERPRETER].append(
+                        (file_name, file_path_str)
+                    )
                     seen_files.add(file_path_str)
 
-        # Add code interpreter file aliases (from --fca) - two-argument explicit naming
+        # Code interpreter file aliases from CLI args
         code_interpreter_file_aliases = args.get(
             "code_interpreter_file_aliases", []
         )
@@ -859,44 +879,26 @@ async def create_template_context_from_routing(
                 and len(name_path_tuple) == 2
             ):
                 custom_name, file_path_raw = name_path_tuple
-                file_path = str(file_path_raw)
-                file_name = str(
-                    custom_name
-                )  # Always use custom name for aliases
-
-                if file_path not in seen_files:
-                    files_tuples.append((file_name, file_path))
-                    seen_files.add(file_path)
-
-        # Also process code_interpreter_files from routing result (for compatibility)
-        for ci_file_item in code_interpreter_files:
-            if isinstance(ci_file_item, (str, Path)):
-                file_name = _generate_template_variable_name(str(ci_file_item))
-                file_path_str = str(ci_file_item)
+                file_path_str = str(file_path_raw)
                 if file_path_str not in seen_files:
-                    files_tuples.append((file_name, file_path_str))
+                    files_by_intent[FileRoutingIntent.CODE_INTERPRETER].append(
+                        (str(custom_name), file_path_str)
+                    )
                     seen_files.add(file_path_str)
-            elif isinstance(ci_file_item, tuple) and len(ci_file_item) == 2:
-                # Handle tuple format (name, path)
-                _, ci_file_path = ci_file_item
-                ci_file_path_str = str(ci_file_path)
-                file_name = _generate_template_variable_name(ci_file_path_str)
-                if ci_file_path_str not in seen_files:
-                    files_tuples.append((file_name, ci_file_path_str))
-                    seen_files.add(ci_file_path_str)
 
-        # Add file search files - now single-argument auto-naming only
+        # File search files from CLI args
         file_search_file_paths = args.get("file_search_files", [])
         for fs_file_path in file_search_file_paths:
             if isinstance(fs_file_path, (str, Path)):
-                # Auto-generate name: -fs docs.pdf
                 file_name = _generate_template_variable_name(str(fs_file_path))
                 file_path_str = str(fs_file_path)
                 if file_path_str not in seen_files:
-                    files_tuples.append((file_name, file_path_str))
+                    files_by_intent[FileRoutingIntent.FILE_SEARCH].append(
+                        (file_name, file_path_str)
+                    )
                     seen_files.add(file_path_str)
 
-        # Add file search file aliases (from --fsa) - two-argument explicit naming
+        # File search file aliases from CLI args
         file_search_file_aliases = args.get("file_search_file_aliases", [])
         for name_path_tuple in file_search_file_aliases:
             if (
@@ -904,79 +906,131 @@ async def create_template_context_from_routing(
                 and len(name_path_tuple) == 2
             ):
                 custom_name, file_path_raw = name_path_tuple
-                file_path = str(file_path_raw)
-                file_name = str(
-                    custom_name
-                )  # Always use custom name for aliases
+                file_path_str = str(file_path_raw)
+                if file_path_str not in seen_files:
+                    files_by_intent[FileRoutingIntent.FILE_SEARCH].append(
+                        (str(custom_name), file_path_str)
+                    )
+                    seen_files.add(file_path_str)
+
+        # Process files from routing result and map to their proper intents
+        # Only add files that haven't been processed from CLI args
+        for template_file_item in template_files:
+            if isinstance(template_file_item, (str, Path)):
+                file_name = _generate_template_variable_name(
+                    str(template_file_item)
+                )
+                file_path_str = str(template_file_item)
+                if file_path_str not in seen_files:
+                    files_by_intent[FileRoutingIntent.TEMPLATE_ONLY].append(
+                        (file_name, file_path_str)
+                    )
+                    seen_files.add(file_path_str)
+            elif (
+                isinstance(template_file_item, tuple)
+                and len(template_file_item) == 2
+            ):
+                _, template_file_path = template_file_item
+                file_path_str = str(template_file_path)
+                file_name = _generate_template_variable_name(file_path_str)
+                if file_path_str not in seen_files:
+                    files_by_intent[FileRoutingIntent.TEMPLATE_ONLY].append(
+                        (file_name, file_path_str)
+                    )
+                    seen_files.add(file_path_str)
 
-                if file_path not in seen_files:
-                    files_tuples.append((file_name, file_path))
-                    seen_files.add(file_path)
+        for ci_file_item in code_interpreter_files:
+            if isinstance(ci_file_item, (str, Path)):
+                file_name = _generate_template_variable_name(str(ci_file_item))
+                file_path_str = str(ci_file_item)
+                if file_path_str not in seen_files:
+                    files_by_intent[FileRoutingIntent.CODE_INTERPRETER].append(
+                        (file_name, file_path_str)
+                    )
+                    seen_files.add(file_path_str)
+            elif isinstance(ci_file_item, tuple) and len(ci_file_item) == 2:
+                _, ci_file_path = ci_file_item
+                file_path_str = str(ci_file_path)
+                file_name = _generate_template_variable_name(file_path_str)
+                if file_path_str not in seen_files:
+                    files_by_intent[FileRoutingIntent.CODE_INTERPRETER].append(
+                        (file_name, file_path_str)
+                    )
+                    seen_files.add(file_path_str)
 
-        # Also process file_search_files from routing result (for compatibility)
         for fs_file_item in file_search_files:
             if isinstance(fs_file_item, (str, Path)):
                 file_name = _generate_template_variable_name(str(fs_file_item))
                 file_path_str = str(fs_file_item)
                 if file_path_str not in seen_files:
-                    files_tuples.append((file_name, file_path_str))
+                    files_by_intent[FileRoutingIntent.FILE_SEARCH].append(
+                        (file_name, file_path_str)
+                    )
                     seen_files.add(file_path_str)
             elif isinstance(fs_file_item, tuple) and len(fs_file_item) == 2:
-                # Handle tuple format (name, path)
                 _, fs_file_path = fs_file_item
-                fs_file_path_str = str(fs_file_path)
-                file_name = _generate_template_variable_name(fs_file_path_str)
-                if fs_file_path_str not in seen_files:
-                    files_tuples.append((file_name, fs_file_path_str))
-                    seen_files.add(fs_file_path_str)
-
-        # Handle directory aliases - create stable template variables for directories
-        dir_mappings = []
+                file_path_str = str(fs_file_path)
+                file_name = _generate_template_variable_name(file_path_str)
+                if file_path_str not in seen_files:
+                    files_by_intent[FileRoutingIntent.FILE_SEARCH].append(
+                        (file_name, file_path_str)
+                    )
+                    seen_files.add(file_path_str)
 
-        # Get directory aliases from routing result (these are already processed from CLI args)
+        # Categorize directories by their intent
         routing = routing_result.routing
         for alias_name, dir_path in routing.template_dir_aliases:
-            dir_mappings.append((alias_name, str(dir_path)))
+            dirs_by_intent[FileRoutingIntent.TEMPLATE_ONLY].append(
+                (alias_name, str(dir_path))
+            )
         for alias_name, dir_path in routing.code_interpreter_dir_aliases:
-            dir_mappings.append((alias_name, str(dir_path)))
+            dirs_by_intent[FileRoutingIntent.CODE_INTERPRETER].append(
+                (alias_name, str(dir_path))
+            )
         for alias_name, dir_path in routing.file_search_dir_aliases:
-            dir_mappings.append((alias_name, str(dir_path)))
+            dirs_by_intent[FileRoutingIntent.FILE_SEARCH].append(
+                (alias_name, str(dir_path))
+            )
 
-        # Auto-naming directories (from -dt, -dc, -ds)
+        # Auto-naming directories from CLI args
         template_dirs = args.get("template_dirs", [])
         for dir_path in template_dirs:
             dir_name = _generate_template_variable_name(str(dir_path))
-            dir_mappings.append((dir_name, str(dir_path)))
+            dirs_by_intent[FileRoutingIntent.TEMPLATE_ONLY].append(
+                (dir_name, str(dir_path))
+            )
 
         code_interpreter_dirs = args.get("code_interpreter_dirs", [])
         for dir_path in code_interpreter_dirs:
             dir_name = _generate_template_variable_name(str(dir_path))
-            dir_mappings.append((dir_name, str(dir_path)))
+            dirs_by_intent[FileRoutingIntent.CODE_INTERPRETER].append(
+                (dir_name, str(dir_path))
+            )
 
         file_search_dirs = args.get("file_search_dirs", [])
         for dir_path in file_search_dirs:
             dir_name = _generate_template_variable_name(str(dir_path))
-            dir_mappings.append((dir_name, str(dir_path)))
+            dirs_by_intent[FileRoutingIntent.FILE_SEARCH].append(
+                (dir_name, str(dir_path))
+            )
 
-        # Process files from explicit routing
-        files_dict = collect_files(
-            file_mappings=cast(
-                List[Tuple[str, Union[str, Path]]], files_tuples
-            ),
-            dir_mappings=cast(
-                List[Tuple[str, Union[str, Path]]], dir_mappings
-            ),
-            dir_recursive=args.get("recursive", False),
-            security_manager=security_manager,
-            routing_type="template",  # Explicitly set routing_type for files processed here
-            # This needs careful thought as files_tuples can come from various sources
-            # For now, we assume files directly added to files_tuples are 'template' routed
-            # if not overridden by a more specific tool routing later.
-            # This is a simplification. A more robust way would be to track routing type
-            # for each path as it's parsed from CLI args.
-            # For the large file warning, FileInfo will default routing_type to None
-            # which FileInfo.content interprets as potentially template-routed.
-        )
+        # Process files by intent groups with appropriate routing_intent
+        for intent, file_mappings in files_by_intent.items():
+            if file_mappings or dirs_by_intent[intent]:
+                intent_files_dict = collect_files(
+                    file_mappings=cast(
+                        List[Tuple[str, Union[str, Path]]], file_mappings
+                    ),
+                    dir_mappings=cast(
+                        List[Tuple[str, Union[str, Path]]],
+                        dirs_by_intent[intent],
+                    ),
+                    dir_recursive=args.get("recursive", False),
+                    security_manager=security_manager,
+                    routing_type="template",  # Keep routing_type for template context accessibility
+                    routing_intent=intent,  # Use intent for warning logic
+                )
+                files_dict.update(intent_files_dict)
 
         # Handle legacy files and directories separately to preserve variable names
         legacy_files = args.get("files", [])
@@ -997,6 +1051,7 @@ async def create_template_context_from_routing(
                 dir_recursive=args.get("recursive", False),
                 security_manager=security_manager,
                 routing_type="template",  # Legacy flags are considered template-only
+                routing_intent=FileRoutingIntent.TEMPLATE_ONLY,  # Legacy files use template-only intent
             )
             # Merge legacy results into the main template context
             files_dict.update(legacy_files_dict)
@@ -1042,8 +1097,18 @@ async def create_template_context_from_routing(
         config = OstructConfig.load(config_path)
         web_search_config = config.get_web_search_config()
 
+        # Apply universal tool toggle overrides (Step 3: Config override hook)
+        enabled_tools: set[str] = args.get("_enabled_tools", set())  # type: ignore[assignment]
+        disabled_tools: set[str] = args.get("_disabled_tools", set())  # type: ignore[assignment]
+
         # Determine if web search should be enabled
-        if web_search_from_cli:
+        if "web-search" in enabled_tools:
+            # Universal --enable-tool web-search takes highest precedence
+            web_search_enabled = True
+        elif "web-search" in disabled_tools:
+            # Universal --disable-tool web-search takes highest precedence
+            web_search_enabled = False
+        elif web_search_from_cli:
             # Explicit --web-search flag takes precedence
             web_search_enabled = True
         elif no_web_search_from_cli:
@@ -1055,6 +1120,69 @@ async def create_template_context_from_routing(
 
         context["web_search_enabled"] = web_search_enabled
 
+        # Add Code Interpreter context variables
+        # Check if Code Interpreter is enabled by looking for CI files or tools
+        ci_enabled_by_routing = bool(
+            args.get("code_interpreter_files")
+            or args.get("code_interpreter_file_aliases")
+            or args.get("code_interpreter_dirs")
+            or args.get("code_interpreter_dir_aliases")
+            or args.get("code_interpreter", False)
+        )
+
+        # Apply universal tool toggle overrides for code-interpreter
+        if "code-interpreter" in enabled_tools:
+            # Universal --enable-tool takes highest precedence
+            code_interpreter_enabled = True
+        elif "code-interpreter" in disabled_tools:
+            # Universal --disable-tool takes highest precedence
+            code_interpreter_enabled = False
+        else:
+            # Fall back to routing-based enablement
+            code_interpreter_enabled = ci_enabled_by_routing
+        context["code_interpreter_enabled"] = code_interpreter_enabled
+
+        # Add auto_download setting from configuration
+        if code_interpreter_enabled:
+            ci_config = config.get_code_interpreter_config()
+            context["auto_download_enabled"] = ci_config.get(
+                "auto_download", True
+            )
+
+            # Determine effective download strategy (config + feature flags)
+            effective_ci_config = dict(
+                ci_config
+            )  # Copy to avoid modifying original
+            enabled_features = args.get("enabled_features", [])
+            disabled_features = args.get("disabled_features", [])
+
+            if enabled_features or disabled_features:
+                try:
+                    from .click_options import parse_feature_flags
+
+                    parsed_flags = parse_feature_flags(
+                        tuple(enabled_features), tuple(disabled_features)
+                    )
+                    ci_hack_flag = parsed_flags.get("ci-download-hack")
+                    if ci_hack_flag == "on":
+                        effective_ci_config["download_strategy"] = (
+                            "two_pass_sentinel"
+                        )
+                    elif ci_hack_flag == "off":
+                        effective_ci_config["download_strategy"] = (
+                            "single_pass"
+                        )
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to parse feature flags in template processor: {e}"
+                    )
+
+            # Add the effective CI config for template processing
+            context["code_interpreter_config"] = effective_ci_config
+        else:
+            context["auto_download_enabled"] = False
+            context["code_interpreter_config"] = {}
+
         return context
 
     except PathSecurityError:

ostruct/cli/template_rendering.py

@@ -56,6 +56,7 @@ Notes:
 
 import logging
 import os
+import re
 from typing import Any, Dict, List, Optional, Union
 
 import jinja2
@@ -95,6 +96,45 @@ TemplateContextValue = Union[
 ]
 
 
+def _extract_variable_name_from_jinja_error(error_message: str) -> str:
+    """Extract the actual variable name from a Jinja2 UndefinedError message.
+
+    Handles various Jinja2 error message formats:
+    - "'variable_name' is undefined"
+    - "'object_description' has no attribute 'property_name'"
+    - Other formats
+
+    Args:
+        error_message: The string representation of the Jinja2 UndefinedError
+
+    Returns:
+        The extracted variable name, or a sanitized version if parsing fails
+    """
+    # Pattern 1: Standard undefined variable: "'variable_name' is undefined"
+    match = re.match(r"'([^']+)' is undefined", error_message)
+    if match:
+        return match.group(1)
+
+    # Pattern 2: Attribute access on object: "'object' has no attribute 'property'"
+    # In this case, we want to extract just the property name, not the object description
+    match = re.match(r"'[^']+' has no attribute '([^']+)'", error_message)
+    if match:
+        property_name = match.group(1)
+        return property_name
+
+    # Pattern 3: Try to find any quoted identifier that looks like a variable name
+    # Look for quoted strings that contain only valid Python identifier characters
+    quoted_parts: List[str] = re.findall(r"'([^']+)'", error_message)
+    for part in quoted_parts:
+        # Check if this looks like a variable name (not a class name or description)
+        if re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", part) and "." not in part:
+            return part
+
+    # Fallback: If we can't parse it properly, return a generic message
+    # This avoids exposing internal class names to users
+    return "unknown_variable"
+
+
 def render_template(
     template_str: str,
     context: Dict[str, Any],
@@ -343,7 +383,7 @@ def render_template(
                 return result
             except jinja2.UndefinedError as e:
                 # Extract variable name from error message
-                var_name = str(e).split("'")[1]
+                var_name = _extract_variable_name_from_jinja_error(str(e))
                 error_msg = (
                     f"Missing required template variable: {var_name}\n"
                     f"Available variables: {', '.join(sorted(context.keys()))}\n"