lionherd-core 1.0.0a3__py3-none-any.whl

lionherd_core/lndl/resolver.py

@@ -0,0 +1,323 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import (
+    BaseModel,
+    ValidationError as PydanticValidationError,
+)
+
+from lionherd_core.libs.schema_handlers._function_call_parser import parse_function_call
+from lionherd_core.types import Operable
+
+from .errors import MissingFieldError, TypeMismatchError
+from .parser import parse_value
+from .types import ActionCall, LactMetadata, LNDLOutput, LvarMetadata
+
+
+def resolve_references_prefixed(
+    out_fields: dict[str, list[str] | str],
+    lvars: dict[str, LvarMetadata],
+    lacts: dict[str, LactMetadata],
+    operable: Operable,
+) -> LNDLOutput:
+    """Resolve namespace-prefixed OUT{} fields and validate against operable specs.
+
+    Args:
+        out_fields: Parsed OUT{} block (field -> list of var names OR literal value)
+        lvars: Extracted namespace-prefixed lvar declarations
+        lacts: Extracted action declarations (name -> LactMetadata)
+        operable: Operable containing allowed specs
+
+    Returns:
+        LNDLOutput with validated Pydantic model instances or scalar values
+
+    Raises:
+        MissingFieldError: Required spec field not in OUT{}
+        TypeMismatchError: Variable model doesn't match spec type
+        ValueError: Variable not found, field mismatch, or name collision
+    """
+    # Check for name collisions between lvars and lacts
+    lvar_names = set(lvars.keys())
+    lact_names = set(lacts.keys())
+    collisions = lvar_names & lact_names
+
+    if collisions:
+        raise ValueError(
+            f"Name collision detected: {collisions} used in both <lvar> and <lact> declarations"
+        )
+    # Check all fields in OUT{} are allowed by operable
+    operable.check_allowed(*out_fields.keys())
+
+    # Check all required specs present
+    for spec in operable.get_specs():
+        is_required = spec.get("required", True)
+        if is_required and spec.name not in out_fields:
+            raise MissingFieldError(f"Required field '{spec.name}' missing from OUT{{}}")
+
+    # Resolve and validate each field (collect all errors)
+    validated_fields = {}
+    parsed_actions: dict[str, ActionCall] = {}  # Actions referenced in OUT{}
+    errors: list[Exception] = []
+
+    for field_name, value in out_fields.items():
+        try:
+            # Get spec for this field
+            spec = operable.get(field_name)
+            if spec is None:
+                raise ValueError(
+                    f"OUT{{}} field '{field_name}' has no corresponding Spec in Operable"
+                )
+
+            # Get type from spec
+            target_type = spec.base_type
+
+            # Check if this is a scalar type (float, str, int, bool)
+            is_scalar = target_type in (float, str, int, bool)
+
+            if is_scalar:
+                # Handle scalar assignment
+                if isinstance(value, list):
+                    # Array syntax for scalar - should be single variable or action
+                    if len(value) != 1:
+                        raise ValueError(
+                            f"Scalar field '{field_name}' cannot use multiple variables, got {value}"
+                        )
+                    var_name = value[0]
+
+                    # Check if this is an action reference
+                    if var_name in lacts:
+                        # Get action metadata
+                        lact_meta = lacts[var_name]
+
+                        # Parse function call with context
+                        try:
+                            parsed_call = parse_function_call(lact_meta.call)
+                        except ValueError as e:
+                            raise ValueError(
+                                f"Invalid function call syntax in action '{var_name}' for scalar field '{field_name}':\n"
+                                f"  Action call: {lact_meta.call}\n"
+                                f"  Parse error: {e}"
+                            ) from e
+
+                        # Create ActionCall instance
+                        action_call = ActionCall(
+                            name=var_name,
+                            function=parsed_call["tool"],
+                            arguments=parsed_call["arguments"],
+                            raw_call=lact_meta.call,
+                        )
+                        parsed_actions[var_name] = action_call
+
+                        # For scalar actions, we mark this field for later execution
+                        # The actual execution happens externally, we just store the action
+                        # For now, we can't validate the result type, so we skip type conversion
+                        # The field will be populated with the action result during execution
+                        validated_fields[field_name] = action_call
+                        continue
+
+                    # Look up variable in lvars
+                    if var_name not in lvars:
+                        raise ValueError(
+                            f"Variable or action '{var_name}' referenced in OUT{{}} but not declared"
+                        )
+
+                    lvar_meta = lvars[var_name]
+                    parsed_value = parse_value(lvar_meta.value)
+                else:
+                    # Literal value (string)
+                    parsed_value = parse_value(value)
+
+                # Type conversion and validation
+                try:
+                    validated_value = target_type(parsed_value)
+                except (ValueError, TypeError) as e:
+                    raise ValueError(
+                        f"Failed to convert value for field '{field_name}' to {target_type.__name__}: {e}"
+                    ) from e
+
+                validated_fields[field_name] = validated_value
+
+            else:
+                # Handle Pydantic BaseModel construction
+                if not isinstance(value, list):
+                    raise ValueError(
+                        f"BaseModel field '{field_name}' requires array syntax, got literal: {value}"
+                    )
+
+                var_list = value
+
+                # Validate it's a BaseModel
+                if not isinstance(target_type, type) or not issubclass(target_type, BaseModel):
+                    raise TypeError(
+                        f"Spec base_type for '{field_name}' must be a Pydantic BaseModel or scalar type, "
+                        f"got {target_type}"
+                    )
+
+                # Special case: single action reference that returns entire model
+                if len(var_list) == 1 and var_list[0] in lacts:
+                    action_name = var_list[0]
+                    lact_meta = lacts[action_name]
+
+                    # Direct actions (no namespace) can return entire model
+                    if lact_meta.model is None:
+                        # Parse function call with context
+                        try:
+                            parsed_call = parse_function_call(lact_meta.call)
+                        except ValueError as e:
+                            raise ValueError(
+                                f"Invalid function call syntax in direct action '{action_name}' for BaseModel field '{field_name}':\n"
+                                f"  Action call: {lact_meta.call}\n"
+                                f"  Parse error: {e}"
+                            ) from e
+
+                        # Create ActionCall instance
+                        action_call = ActionCall(
+                            name=action_name,
+                            function=parsed_call["tool"],
+                            arguments=parsed_call["arguments"],
+                            raw_call=lact_meta.call,
+                        )
+                        parsed_actions[action_name] = action_call
+
+                        # Store action as field value (will be executed to get model instance)
+                        validated_fields[field_name] = action_call
+                        continue
+                    # If namespaced, fall through to mixing logic below
+
+                # Build kwargs from variable list - supports mixing lvars and namespaced actions
+                kwargs = {}
+                for var_name in var_list:
+                    # Check if this is an action reference
+                    if var_name in lacts:
+                        lact_meta = lacts[var_name]
+
+                        # Namespaced actions must specify which field they populate
+                        if lact_meta.model is None or lact_meta.field is None:
+                            raise ValueError(
+                                f"Direct action '{var_name}' cannot be mixed with lvars in BaseModel field '{field_name}'. "
+                                f"Use namespaced syntax: <lact {target_type.__name__}.fieldname {var_name}>...</lact>"
+                            )
+
+                        # Validate: model name matches
+                        if lact_meta.model != target_type.__name__:
+                            raise TypeMismatchError(
+                                f"Action '{var_name}' is for model '{lact_meta.model}', "
+                                f"but field '{field_name}' expects '{target_type.__name__}'"
+                            )
+
+                        # Parse action function call with context
+                        try:
+                            parsed_call = parse_function_call(lact_meta.call)
+                        except ValueError as e:
+                            raise ValueError(
+                                f"Invalid function call syntax in namespaced action '{var_name}' for field '{lact_meta.model}.{lact_meta.field}':\n"
+                                f"  Action call: {lact_meta.call}\n"
+                                f"  Parse error: {e}"
+                            ) from e
+
+                        # Create ActionCall instance
+                        action_call = ActionCall(
+                            name=var_name,
+                            function=parsed_call["tool"],
+                            arguments=parsed_call["arguments"],
+                            raw_call=lact_meta.call,
+                        )
+                        parsed_actions[var_name] = action_call
+
+                        # Use the namespaced field to map action result
+                        kwargs[lact_meta.field] = action_call
+                        continue
+
+                    # Look up variable in lvars
+                    if var_name not in lvars:
+                        raise ValueError(
+                            f"Variable or action '{var_name}' referenced in OUT{{}} but not declared"
+                        )
+
+                    lvar_meta = lvars[var_name]
+
+                    # Validate: model name matches
+                    if lvar_meta.model != target_type.__name__:
+                        raise TypeMismatchError(
+                            f"Variable '{var_name}' is for model '{lvar_meta.model}', "
+                            f"but field '{field_name}' expects '{target_type.__name__}'"
+                        )
+
+                    # Map field name to kwargs
+                    kwargs[lvar_meta.field] = parse_value(lvar_meta.value)
+
+                # Construct Pydantic model instance
+                # WARNING: model_construct() bypasses Pydantic validation when ActionCall objects present.
+                # Caller MUST re-validate after executing actions using revalidate_with_action_results().
+                # See LNDLOutput docstring for complete action execution lifecycle.
+                has_actions = any(isinstance(v, ActionCall) for v in kwargs.values())
+                try:
+                    if has_actions:
+                        # PARTIAL VALIDATION: Field constraints, validators, and type checking bypassed
+                        instance = target_type.model_construct(**kwargs)
+                    else:
+                        # FULL VALIDATION: Normal Pydantic validation for models without actions
+                        instance = target_type(**kwargs)
+                except PydanticValidationError as e:
+                    raise ValueError(
+                        f"Failed to construct {target_type.__name__} for field '{field_name}': {e}"
+                    ) from e
+
+                # Apply validators/rules if specified in spec metadata
+                validators = spec.get("validator")
+                if validators:
+                    validators = validators if isinstance(validators, list) else [validators]
+                    for validator in validators:
+                        if hasattr(validator, "invoke"):
+                            instance = validator.invoke(field_name, instance, target_type)
+                        else:
+                            instance = validator(instance)
+
+                validated_fields[field_name] = instance
+
+        except Exception as e:
+            # Collect errors for aggregation
+            errors.append(e)
+
+    # Raise all collected errors as ExceptionGroup
+    if errors:
+        raise ExceptionGroup("LNDL validation failed", errors)
+
+    return LNDLOutput(
+        fields=validated_fields,
+        lvars=lvars,
+        lacts=lacts,
+        actions=parsed_actions,
+        raw_out_block=str(out_fields),
+    )
+
+
+def parse_lndl(response: str, operable: Operable) -> LNDLOutput:
+    """Parse LNDL response and validate against operable specs.
+
+    Args:
+        response: Full LLM response containing lvars, lacts, and OUT{}
+        operable: Operable containing allowed specs
+
+    Returns:
+        LNDLOutput with validated fields and parsed actions
+    """
+    from .parser import (
+        extract_lacts_prefixed,
+        extract_lvars_prefixed,
+        extract_out_block,
+        parse_out_block_array,
+    )
+
+    # 1. Extract namespace-prefixed lvars
+    lvars_prefixed = extract_lvars_prefixed(response)
+
+    # 2. Extract action declarations (with namespace support)
+    lacts_prefixed = extract_lacts_prefixed(response)
+
+    # 3. Extract and parse OUT{} block with array syntax
+    out_content = extract_out_block(response)
+    out_fields = parse_out_block_array(out_content)
+
+    # 4. Resolve references and validate
+    return resolve_references_prefixed(out_fields, lvars_prefixed, lacts_prefixed, operable)

lionherd_core/lndl/types.py

@@ -0,0 +1,287 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+"""Core types for LNDL (Lion Directive Language)."""
+
+from dataclasses import dataclass
+from typing import Any
+
+from pydantic import BaseModel
+
+
+@dataclass(slots=True, frozen=True)
+class LvarMetadata:
+    """Metadata for namespace-prefixed lvar.
+
+    Example: <lvar Report.title title>Good Title</lvar>
+    → LvarMetadata(model="Report", field="title", local_name="title", value="Good Title")
+    """
+
+    model: str  # Model name (e.g., "Report")
+    field: str  # Field name (e.g., "title")
+    local_name: str  # Local variable name (e.g., "title")
+    value: str  # Raw string value
+
+
+@dataclass(slots=True, frozen=True)
+class LactMetadata:
+    """Metadata for action declaration (namespaced or direct).
+
+    Examples:
+        Namespaced: <lact Report.summary s>generate_summary(...)</lact>
+        → LactMetadata(model="Report", field="summary", local_name="s", call="generate_summary(...)")
+
+        Direct: <lact search>search(...)</lact>
+        → LactMetadata(model=None, field=None, local_name="search", call="search(...)")
+    """
+
+    model: str | None  # Model name (e.g., "Report") or None for direct actions
+    field: str | None  # Field name (e.g., "summary") or None for direct actions
+    local_name: str  # Local reference name (e.g., "s", "search")
+    call: str  # Raw function call string
+
+
+@dataclass(slots=True, frozen=True)
+class ParsedConstructor:
+    """Parsed type constructor from OUT{} block."""
+
+    class_name: str
+    kwargs: dict[str, Any]
+    raw: str
+
+    @property
+    def has_dict_unpack(self) -> bool:
+        """Check if constructor uses **dict unpacking."""
+        return any(k.startswith("**") for k in self.kwargs)
+
+
+@dataclass(slots=True, frozen=True)
+class ActionCall:
+    """Parsed action call from <lact> tag.
+
+    Represents a tool/function invocation declared in LNDL response.
+    Actions are only executed if referenced in OUT{} block.
+
+    Attributes:
+        name: Local reference name (e.g., "search", "validate")
+        function: Function/tool name to invoke
+        arguments: Parsed arguments dict
+        raw_call: Original Python function call string
+    """
+
+    name: str
+    function: str
+    arguments: dict[str, Any]
+    raw_call: str
+
+
+@dataclass(slots=True, frozen=True)
+class LNDLOutput:
+    """Validated LNDL output with action execution lifecycle.
+
+    Action Execution Lifecycle:
+    ---------------------------
+    1. **Parse**: LNDL response parsed, ActionCall objects created for referenced actions
+    2. **Partial Validation**: BaseModels with ActionCall fields use model_construct() to bypass validation
+    3. **Execute**: Caller executes actions using .actions dict, collects results
+    4. **Re-validate**: Caller replaces ActionCall objects with results and re-validates models
+
+    Fields containing ActionCall objects have **partial validation** only:
+    - Field constraints (validators, bounds, regex) are NOT enforced
+    - Type checking is bypassed
+    - Re-validation MUST occur after action execution
+
+    Example:
+        >>> output = parse_lndl(response, operable)
+        >>> # Execute actions
+        >>> action_results = {}
+        >>> for name, action in output.actions.items():
+        >>>     result = execute_tool(action.function, action.arguments)
+        >>>     action_results[name] = result
+        >>>
+        >>> # Re-validate models with action results
+        >>> for field_name, value in output.fields.items():
+        >>>     if isinstance(value, BaseModel) and has_action_calls(value):
+        >>>         value = revalidate_with_action_results(value, action_results)
+        >>>         output.fields[field_name] = value
+    """
+
+    fields: dict[str, BaseModel | ActionCall]  # BaseModel instances or ActionCall (pre-execution)
+    lvars: dict[str, str] | dict[str, LvarMetadata]  # Preserved for debugging
+    lacts: dict[str, LactMetadata]  # All declared actions (for debugging/reference)
+    actions: dict[str, ActionCall]  # Actions referenced in OUT{} (pending execution)
+    raw_out_block: str  # Preserved for debugging
+
+    def __getitem__(self, key: str) -> BaseModel | ActionCall:
+        return self.fields[key]
+
+    def __getattr__(self, key: str) -> BaseModel | ActionCall:
+        if key in ("fields", "lvars", "lacts", "actions", "raw_out_block"):
+            return object.__getattribute__(self, key)
+        return self.fields[key]
+
+
+def has_action_calls(model: BaseModel) -> bool:
+    """Check if a BaseModel instance contains any ActionCall objects in its fields.
+
+    Recursively checks nested BaseModel fields and collection fields (list, dict, tuple, set)
+    to detect ActionCall objects at any depth.
+
+    Args:
+        model: Pydantic BaseModel instance to check
+
+    Returns:
+        True if any field value is an ActionCall (at any nesting level), False otherwise
+
+    Example:
+        >>> report = Report.model_construct(title="Report", summary=ActionCall(...))
+        >>> has_action_calls(report)
+        True
+        >>> # Also detects in nested models
+        >>> nested = NestedReport(main=report)
+        >>> has_action_calls(nested)
+        True
+    """
+
+    def _check_value(value: Any) -> bool:
+        """Recursively check a value for ActionCall objects."""
+        # Direct ActionCall
+        if isinstance(value, ActionCall):
+            return True
+
+        # Nested BaseModel - recurse
+        if isinstance(value, BaseModel):
+            return has_action_calls(value)
+
+        # Collections - check items
+        if isinstance(value, (list, tuple, set)):
+            return any(_check_value(item) for item in value)
+
+        if isinstance(value, dict):
+            return any(_check_value(v) for v in value.values())
+
+        return False
+
+    return any(_check_value(value) for value in model.__dict__.values())
+
+
+def ensure_no_action_calls(model: BaseModel) -> BaseModel:
+    """Validate that model contains no unexecuted ActionCall objects.
+
+    Use this guard before persisting models to prevent database corruption or logic errors.
+    Models with ActionCall placeholders must be re-validated with action results first.
+
+    Recursively checks nested models and collections for ActionCall objects.
+
+    Args:
+        model: BaseModel instance to validate
+
+    Returns:
+        The same model instance if validation passes
+
+    Raises:
+        ValueError: If model contains any ActionCall objects, with field path details
+
+    Example:
+        >>> # CRITICAL: Always guard before persistence
+        >>> output = parse_lndl_fuzzy(llm_response, operable)
+        >>> report = output.report
+        >>>
+        >>> # Execute actions first
+        >>> action_results = execute_actions(output.actions)
+        >>> validated_report = revalidate_with_action_results(report, action_results)
+        >>>
+        >>> # Safe to persist - guard will pass
+        >>> db.save(ensure_no_action_calls(validated_report))
+        >>>
+        >>> # BAD: Forgot revalidation - guard prevents corruption
+        >>> db.save(ensure_no_action_calls(report))  # Raises ValueError!
+    """
+
+    def _find_action_call_fields(obj: Any, path: str = "") -> list[str]:
+        """Find all field paths containing ActionCall objects."""
+        paths = []
+
+        if isinstance(obj, ActionCall):
+            return [path] if path else ["<root>"]
+
+        if isinstance(obj, BaseModel):
+            for field_name, value in obj.__dict__.items():
+                field_path = f"{path}.{field_name}" if path else field_name
+                paths.extend(_find_action_call_fields(value, field_path))
+
+        elif isinstance(obj, (list, tuple, set)):
+            for idx, item in enumerate(obj):
+                item_path = f"{path}[{idx}]"
+                paths.extend(_find_action_call_fields(item, item_path))
+
+        elif isinstance(obj, dict):
+            for key, value in obj.items():
+                dict_path = f"{path}[{key!r}]"
+                paths.extend(_find_action_call_fields(value, dict_path))
+
+        return paths
+
+    if has_action_calls(model):
+        model_name = type(model).__name__
+        action_call_fields = _find_action_call_fields(model)
+        fields_str = ", ".join(action_call_fields[:3])  # Show first 3 fields
+        if len(action_call_fields) > 3:
+            fields_str += f" (and {len(action_call_fields) - 3} more)"
+
+        raise ValueError(
+            f"{model_name} contains unexecuted actions in fields: {fields_str}. "
+            f"Models with ActionCall placeholders must be re-validated after action execution. "
+            f"Call revalidate_with_action_results() before using this model."
+        )
+    return model
+
+
+def revalidate_with_action_results(
+    model: BaseModel,
+    action_results: dict[str, Any],
+) -> BaseModel:
+    """Replace ActionCall fields with execution results and re-validate the model.
+
+    This function must be called after executing actions to restore full Pydantic validation.
+    Models constructed with model_construct() have bypassed validation and may contain
+    ActionCall objects where actual values are expected.
+
+    Args:
+        model: BaseModel instance with ActionCall placeholders
+        action_results: Dict mapping action names to their execution results
+
+    Returns:
+        Fully validated BaseModel instance with action results substituted
+
+    Raises:
+        ValidationError: If action results don't satisfy field constraints
+
+    Example:
+        >>> # Model has ActionCall in summary field
+        >>> report = Report.model_construct(title="Report", summary=action_call)
+        >>>
+        >>> # Execute action and get result
+        >>> action_results = {"summarize": "Generated summary text"}
+        >>>
+        >>> # Re-validate with results
+        >>> validated_report = revalidate_with_action_results(report, action_results)
+        >>> isinstance(validated_report.summary, str)  # True, no longer ActionCall
+        True
+    """
+    # Get current field values
+    kwargs = model.model_dump()
+
+    # Replace ActionCall objects with their execution results
+    for field_name, value in model.__dict__.items():
+        if isinstance(value, ActionCall):
+            # Find result by action name
+            if value.name not in action_results:
+                raise ValueError(
+                    f"Action '{value.name}' in field '{field_name}' has no execution result. "
+                    f"Available results: {list(action_results.keys())}"
+                )
+            kwargs[field_name] = action_results[value.name]
+
+    # Re-construct with full validation
+    return type(model)(**kwargs)