ostruct-cli 0.1.0__py3-none-any.whl

ostruct/cli/template_utils.py

@@ -0,0 +1,288 @@
+"""Template utilities for the CLI.
+
+This module serves as the main entry point for template processing functionality.
+It re-exports the public APIs from specialized modules:
+
+- template_schema: Schema validation and proxy objects
+- template_validation: Template validation using Jinja2
+- template_rendering: Template rendering with Jinja2
+- template_io: File I/O operations and metadata extraction
+"""
+
+from typing import Any, Dict, List, Optional, Set
+
+import jsonschema
+from jinja2 import Environment, meta
+from jinja2.nodes import Node
+
+from .errors import (
+    CLIError,
+    SchemaError,
+    SchemaValidationError,
+    SystemPromptError,
+    TaskTemplateError,
+    TaskTemplateSyntaxError,
+    TaskTemplateVariableError,
+)
+from .file_utils import FileInfo
+from .template_io import extract_metadata, extract_template_metadata, read_file
+from .template_rendering import DotDict, render_template
+from .template_schema import (
+    DictProxy,
+    FileInfoProxy,
+    ValidationProxy,
+    create_validation_context,
+)
+from .template_validation import SafeUndefined, validate_template_placeholders
+
+
+# Custom error classes
+class TemplateMetadataError(TaskTemplateError):
+    """Raised when there are issues extracting template metadata."""
+
+    pass
+
+
+def validate_json_schema(schema: Dict[str, Any]) -> None:
+    """Validate that a dictionary follows JSON Schema structure.
+
+    This function checks that the provided dictionary is a valid JSON Schema,
+    following the JSON Schema specification.
+
+    Args:
+        schema: Dictionary to validate as a JSON Schema
+
+    Raises:
+        SchemaValidationError: If the schema is invalid
+    """
+    try:
+        # Get the validator class for the schema
+        validator_cls = jsonschema.validators.validator_for(schema)
+
+        # Check schema itself is valid
+        validator_cls.check_schema(schema)
+
+        # Create validator instance
+        validator_cls(schema)
+    except jsonschema.exceptions.SchemaError as e:
+        raise SchemaValidationError(f"Invalid JSON Schema: {e}")
+    except Exception as e:
+        raise SchemaValidationError(f"Schema validation error: {e}")
+
+
+def validate_response(
+    response: Dict[str, Any], schema: Dict[str, Any]
+) -> None:
+    """Validate that a response dictionary matches a JSON Schema.
+
+    This function validates that the response dictionary conforms to the provided
+    JSON Schema specification.
+
+    Args:
+        response: Dictionary to validate
+        schema: JSON Schema to validate against
+
+    Raises:
+        ValueError: If the response does not match the schema
+    """
+    try:
+        # Create a validator for the schema
+        validator = jsonschema.validators.validator_for(schema)(schema)
+
+        # Validate the response
+        validator.validate(response)
+    except jsonschema.exceptions.ValidationError as e:
+        raise ValueError(f"Response validation failed: {e}")
+    except Exception as e:
+        raise ValueError(f"Response validation error: {e}")
+
+
+def find_all_template_variables(
+    template: str, env: Optional[Environment] = None
+) -> Set[str]:
+    """Find all variables used in a template and its dependencies.
+
+    This function recursively parses the template and any included/imported/extended
+    templates to find all variables that might be used.
+
+    Args:
+        template: The template string to parse
+        env: Optional Jinja2 environment. If not provided, a new one will be created.
+
+    Returns:
+        Set of all variable names used in the template and its dependencies
+
+    Raises:
+        jinja2.TemplateSyntaxError: If the template has invalid syntax
+    """
+    if env is None:
+        env = Environment(undefined=SafeUndefined)
+
+    # Parse template
+    ast = env.parse(template)
+
+    # Find all variables in this template
+    variables = meta.find_undeclared_variables(ast)
+
+    # Filter out built-in functions and filters
+    builtin_vars = {
+        # Core functions
+        "range",
+        "dict",
+        "lipsum",
+        "cycler",
+        "joiner",
+        "namespace",
+        "loop",
+        "super",
+        "self",
+        "varargs",
+        "kwargs",
+        "items",
+        # String filters
+        "upper",
+        "lower",
+        "title",
+        "capitalize",
+        "trim",
+        "strip",
+        "lstrip",
+        "rstrip",
+        "center",
+        "ljust",
+        "rjust",
+        "wordcount",
+        "truncate",
+        "striptags",
+        # List filters
+        "first",
+        "last",
+        "length",
+        "max",
+        "min",
+        "sum",
+        "sort",
+        "unique",
+        "reverse",
+        "reject",
+        "select",
+        "map",
+        "join",
+        "count",
+        # Type conversion
+        "abs",
+        "round",
+        "int",
+        "float",
+        "string",
+        "list",
+        "bool",
+        "batch",
+        "slice",
+        "attr",
+        # Common filters
+        "default",
+        "replace",
+        "safe",
+        "urlencode",
+        "indent",
+        "format",
+        "escape",
+        "e",
+        "nl2br",
+        "urlize",
+        # Dictionary operations
+        "items",
+        "keys",
+        "values",
+        "dictsort",
+        # Math operations
+        "round",
+        "ceil",
+        "floor",
+        "abs",
+        "max",
+        "min",
+        # Date/time
+        "now",
+        "strftime",
+        "strptime",
+        "datetimeformat",
+    }
+
+    variables = variables - builtin_vars
+
+    # Find template dependencies (include, extends, import)
+    def visit_nodes(nodes: List[Node]) -> None:
+        """Visit AST nodes recursively to find template dependencies.
+
+        Args:
+            nodes: List of AST nodes to visit
+        """
+        for node in nodes:
+            if node.__class__.__name__ in (
+                "Include",
+                "Extends",
+                "Import",
+                "FromImport",
+            ):
+                # Get the template name
+                if hasattr(node, "template"):
+                    template_name = node.template.value
+                    try:
+                        # Load and parse the referenced template
+                        if env.loader is not None:
+                            included_template = env.loader.get_source(
+                                env, template_name
+                            )[0]
+                            # Recursively find variables in the included template
+                            variables.update(
+                                find_all_template_variables(
+                                    included_template, env
+                                )
+                            )
+                    except Exception:
+                        # Skip if template can't be loaded - it will be caught during rendering
+                        pass
+
+            # Recursively visit child nodes
+            if hasattr(node, "body"):
+                visit_nodes(node.body)
+            if hasattr(node, "else_"):
+                visit_nodes(node.else_)
+
+    visit_nodes(ast.body)
+    return variables  # type: ignore[no-any-return]
+
+
+__all__ = [
+    # Schema types and validation
+    "ValidationProxy",
+    "FileInfoProxy",
+    "DictProxy",
+    "create_validation_context",
+    "validate_json_schema",
+    "validate_response",
+    # Template validation
+    "SafeUndefined",
+    "validate_template_placeholders",
+    "find_all_template_variables",
+    # Template rendering
+    "render_template",
+    "DotDict",
+    # File I/O
+    "read_file",
+    "extract_metadata",
+    "extract_template_metadata",
+    # File info
+    "FileInfo",
+    # Error classes
+    "CLIError",
+    "TaskTemplateError",
+    "TaskTemplateSyntaxError",
+    "TaskTemplateVariableError",
+    "SchemaError",
+    "SchemaValidationError",
+    "SystemPromptError",
+    "TemplateMetadataError",
+]

ostruct/cli/template_validation.py

@@ -0,0 +1,375 @@
+"""Template validation using Jinja2.
+
+This module provides functionality for validating Jinja2 templates, ensuring that:
+1. All variables used in templates exist in the context
+2. Nested attribute/key access is valid according to schema
+3. Loop variables and filters are used correctly
+
+Key Components:
+    - SafeUndefined: Custom undefined type for validation
+    - validate_template_placeholders: Main validation function
+
+Examples:
+    Basic template validation:
+    >>> template = "Hello {{ name }}, your score is {{ results.score }}"
+    >>> template_context = {'name': 'value', 'results': {'score': 100}}
+    >>> validate_template_placeholders(template, template_context)  # OK
+    >>>
+    >>> # Missing variable raises ValueError:
+    >>> template = "Hello {{ missing }}"
+    >>> validate_template_placeholders(template, {'name'})  # Raises ValueError
+
+    Nested attribute validation:
+    >>> template = '''
+    ... Debug mode: {{ config.debug }}
+    ... Settings: {{ config.settings.mode }}
+    ... Invalid: {{ config.invalid }}
+    ... '''
+    >>> validate_template_placeholders(template, {'config'})  # Raises ValueError
+
+    Loop variable validation:
+    >>> template = '''
+    ... {% for item in items %}
+    ...   - {{ item.name }}: {{ item.value }}
+    ...   Invalid: {{ item.invalid }}
+    ... {% endfor %}
+    ... '''
+    >>> validate_template_placeholders(template, {'items'})  # Raises ValueError
+
+    Filter validation:
+    >>> template = '''
+    ... {{ code | format_code }}  {# OK - valid filter #}
+    ... {{ data | invalid_filter }}  {# Raises ValueError #}
+    ... '''
+    >>> validate_template_placeholders(template, {'code', 'data'})
+
+Notes:
+    - Uses Jinja2's meta API to find undeclared variables
+    - Supports custom filters through safe wrappers
+    - Provides detailed error messages for validation failures
+"""
+
+import logging
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Set,
+    TypeVar,
+    Union,
+    cast,
+)
+
+import jinja2
+from jinja2 import meta
+from jinja2.nodes import For, Name, Node
+
+from . import template_filters
+from .errors import TemplateValidationError
+from .template_env import create_jinja_env
+from .template_schema import (
+    DictProxy,
+    FileInfoProxy,
+    ValidationProxy,
+    create_validation_context,
+)
+
+T = TypeVar("T")
+FilterFunc = Callable[..., Any]
+FilterWrapper = Callable[[Any, Any, Any], Optional[Union[Any, str, List[Any]]]]
+
+__all__ = [
+    "TemplateValidationError",
+    "validate_template_placeholders",
+    "validate_template_file",
+]
+
+
+class SafeUndefined(jinja2.StrictUndefined):  # type: ignore[misc]
+    """A strict Undefined class that validates attribute access during validation."""
+
+    def __getattr__(self, name: str) -> Any:
+        # Raise error for attribute access on undefined values
+        if name not in {"__html__", "__html_format__"}:
+            raise jinja2.UndefinedError(
+                f"'{self._undefined_name}' has no attribute '{name}'"
+            )
+        return self
+
+    def __getitem__(self, key: Any) -> Any:
+        # Raise error for key access on undefined values
+        raise jinja2.UndefinedError(
+            f"'{self._undefined_name}' has no key '{key}'"
+        )
+
+
+def safe_filter(func: FilterFunc) -> FilterWrapper:
+    """Wrap a filter function to handle None and proxy values safely."""
+
+    def wrapper(
+        value: Any, *args: Any, **kwargs: Any
+    ) -> Optional[Union[Any, str, List[Any]]]:
+        if value is None:
+            return None
+        if isinstance(value, (ValidationProxy, FileInfoProxy, DictProxy)):
+            # For validation, just return an empty result of the appropriate type
+            if func.__name__ in (
+                "extract_field",
+                "frequency",
+                "aggregate",
+                "pivot_table",
+                "summarize",
+            ):
+                return []
+            elif func.__name__ in ("dict_to_table", "list_to_table"):
+                return ""
+            return value
+        return func(value, *args, **kwargs)
+
+    return wrapper
+
+
+def find_loop_vars(nodes: List[Node]) -> Set[str]:
+    """Find variables used in loop constructs."""
+    loop_vars = set()
+    for node in nodes:
+        if isinstance(node, For):
+            target = node.target
+            if isinstance(target, Name):
+                loop_vars.add(target.name)
+            elif hasattr(target, "items"):
+                items = cast(List[Name], target.items)
+                for item in items:
+                    loop_vars.add(item.name)
+        if hasattr(node, "body"):
+            loop_vars.update(find_loop_vars(cast(List[Node], node.body)))
+        if hasattr(node, "else_"):
+            loop_vars.update(find_loop_vars(cast(List[Node], node.else_)))
+    return loop_vars
+
+
+def validate_template_placeholders(
+    template: str, template_context: Optional[Dict[str, Any]] = None
+) -> None:
+    """Validate that all placeholders in a template are valid.
+
+    Args:
+        template: Template string to validate
+        template_context: Optional context to validate against
+
+    Raises:
+        TemplateValidationError: If any placeholders are invalid
+    """
+    logger = logging.getLogger(__name__)
+
+    logger.debug("=== validate_template_placeholders called ===")
+    logger.debug(
+        "Args: template=%s, template_context=%s",
+        template,
+        {
+            k: type(v).__name__
+            for k, v in (template_context or {}).items()
+            if v is not None
+        },
+    )
+
+    try:
+        # 1) Create Jinja2 environment with meta extension and safe undefined
+        env = create_jinja_env(validation_mode=True)
+
+        # Register custom filters with None-safe wrappers
+        env.filters.update(
+            {
+                "format_code": safe_filter(template_filters.format_code),
+                "strip_comments": safe_filter(template_filters.strip_comments),
+                "extract_field": safe_filter(template_filters.extract_field),
+                "frequency": safe_filter(template_filters.frequency),
+                "aggregate": safe_filter(template_filters.aggregate),
+                "pivot_table": safe_filter(template_filters.pivot_table),
+                "summarize": safe_filter(template_filters.summarize),
+                "dict_to_table": safe_filter(template_filters.dict_to_table),
+                "list_to_table": safe_filter(template_filters.list_to_table),
+            }
+        )
+
+        # Add built-in Jinja2 functions and filters
+        builtin_vars = {
+            # Core functions
+            "range",
+            "dict",
+            "lipsum",
+            "cycler",
+            "joiner",
+            "namespace",
+            "loop",
+            "super",
+            "self",
+            "varargs",
+            "kwargs",
+            "items",
+            # String filters
+            "upper",
+            "lower",
+            "title",
+            "capitalize",
+            "trim",
+            "strip",
+            "lstrip",
+            "rstrip",
+            "center",
+            "ljust",
+            "rjust",
+            "wordcount",
+            "truncate",
+            "striptags",
+            # List filters
+            "first",
+            "last",
+            "length",
+            "max",
+            "min",
+            "sum",
+            "sort",
+            "unique",
+            "reverse",
+            "reject",
+            "select",
+            "map",
+            "join",
+            "count",
+            # Type conversion
+            "abs",
+            "round",
+            "int",
+            "float",
+            "string",
+            "list",
+            "bool",
+            "batch",
+            "slice",
+            "attr",
+            # Common filters
+            "default",
+            "replace",
+            "safe",
+            "urlencode",
+            "indent",
+            "format",
+            "escape",
+            "e",
+            "nl2br",
+            "urlize",
+            # Dictionary operations
+            "items",
+            "keys",
+            "values",
+            "dictsort",
+            # Math operations
+            "round",
+            "ceil",
+            "floor",
+            "abs",
+            "max",
+            "min",
+            # Date/time
+            "now",
+            "strftime",
+            "strptime",
+            "datetimeformat",
+        }
+        builtin_vars.update(env.filters.keys())
+        builtin_vars.update(env.globals.keys())
+
+        # 2) Parse template and find variables
+        ast = env.parse(template)
+        available_vars = (
+            set(template_context.keys()) if template_context else set()
+        )
+        logger.debug("Available variables: %s", available_vars)
+
+        # Find loop variables
+        loop_vars = find_loop_vars(ast.body)
+        logger.debug("Found loop variables: %s", loop_vars)
+
+        # Find all undeclared variables using jinja2.meta
+        found_vars = meta.find_undeclared_variables(ast)
+        logger.debug("Found undeclared variables: %s", found_vars)
+
+        # Check for missing variables
+        missing = {
+            name
+            for name in found_vars
+            if name not in available_vars
+            and name not in builtin_vars
+            and name not in loop_vars
+            and name
+            != "stdin"  # Special case for stdin which may be added dynamically
+        }
+
+        if missing:
+            raise TemplateValidationError(
+                f"Task template uses undefined variables: {', '.join(sorted(missing))}. "
+                f"Available variables: {', '.join(sorted(available_vars))}"
+            )
+
+        logger.debug(
+            "Before create_validation_context - available_vars type: %s, value: %s",
+            type(available_vars),
+            available_vars,
+        )
+        logger.debug(
+            "Before create_validation_context - template_context type: %s, value: %s",
+            type(template_context),
+            template_context,
+        )
+
+        # 3) Create validation context with proxy objects
+        logger.debug(
+            "Creating validation context with template_context: %s",
+            template_context,
+        )
+        validation_context = create_validation_context(template_context or {})
+
+        # 4) Try to render with validation context
+        try:
+            env.from_string(template).render(validation_context)
+        except jinja2.UndefinedError as e:
+            # Convert Jinja2 undefined errors to our own ValueError
+            raise TemplateValidationError(str(e))
+        except ValueError as e:
+            # Convert validation errors from template_schema to TemplateValidationError
+            raise TemplateValidationError(str(e))
+        except Exception as e:
+            logger.error(
+                "Unexpected error during template validation: %s", str(e)
+            )
+            raise
+
+    except jinja2.TemplateSyntaxError as e:
+        # Convert Jinja2 syntax errors to our own ValueError
+        raise TemplateValidationError(
+            f"Invalid task template syntax: {str(e)}"
+        )
+    except Exception as e:
+        logger.error("Unexpected error during template validation: %s", str(e))
+        raise
+
+
+def validate_template_file(
+    template_path: str,
+    template_context: Optional[Dict[str, Any]] = None,
+) -> None:
+    """Validate a template file with the given context.
+
+    Args:
+        template_path: Path to the template file
+        template_context: Optional dictionary containing template variables
+
+    Raises:
+        TemplateValidationError: If template validation fails
+    """
+    with open(template_path, "r", encoding="utf-8") as f:
+        template_str = f.read()
+    validate_template_placeholders(template_str, template_context)

ostruct/cli/utils.py

@@ -0,0 +1,31 @@
+"""Common utilities for the CLI package."""
+
+from typing import Tuple
+
+from .errors import VariableNameError, VariableValueError
+
+
+def parse_mapping(mapping: str) -> Tuple[str, str]:
+    """Parse a mapping string in the format 'name=value'.
+
+    Args:
+        mapping: Mapping string in format 'name=value'
+
+    Returns:
+        Tuple of (name, value)
+
+    Raises:
+        ValueError: If mapping format is invalid
+        VariableNameError: If name part is empty
+        VariableValueError: If value part is empty
+    """
+    if not mapping or "=" not in mapping:
+        raise ValueError("Invalid mapping format")
+
+    name, value = mapping.split("=", 1)
+    if not name:
+        raise VariableNameError("Empty name in mapping")
+    if not value:
+        raise VariableValueError("Empty value in mapping")
+
+    return name, value