soe-ai 0.2.0b1__py3-none-any.whl

soe/init.py

@@ -0,0 +1,165 @@
+"""
+Convenience initialization for SOE.
+
+This module provides easy setup functions for common use cases.
+Use these to quickly get started without manually wiring nodes and backends.
+"""
+
+import copy
+from typing import Callable, Dict, Any, Tuple, Optional, List, Union
+from .broker import broadcast_signals, orchestrate
+from .nodes.router.factory import create_router_node_caller
+from .nodes.llm.factory import create_llm_node_caller
+from .nodes.agent.factory import create_agent_node_caller
+from .nodes.tool.factory import create_tool_node_caller
+from .nodes.child.factory import create_child_node_caller
+from .lib.yaml_parser import parse_yaml
+from .types import CallLlm, Backends, NodeCaller, BroadcastSignalsCaller
+from .local_backends import create_in_memory_backends, create_local_backends
+
+
+def create_all_nodes(
+    backends: Backends,
+    call_llm: Optional[CallLlm] = None,
+    tools_registry: Optional[Dict[str, Callable]] = None,
+) -> Tuple[Dict[str, NodeCaller], BroadcastSignalsCaller]:
+    """
+    Create all node types with automatic wiring.
+
+    This is the recommended way to set up nodes for orchestration.
+    Returns both the nodes dictionary and the broadcast_signals_caller.
+
+    Args:
+        backends: Backend services (use create_in_memory_backends or create_local_backends)
+        call_llm: Optional LLM caller function for LLM/Agent nodes
+        tools_registry: Optional dict mapping tool name -> callable for Tool/Agent nodes
+
+    Returns:
+        Tuple of (nodes dict, broadcast_signals_caller function)
+
+    Example:
+        backends = create_in_memory_backends()
+        nodes, broadcast = create_all_nodes(backends, call_llm=my_llm, tools_registry=my_tools)
+
+        execution_id = orchestrate(
+            config=workflow_yaml,
+            initial_workflow_name="my_workflow",
+            initial_signals=["START"],
+            initial_context={"user_input": "Hello"},
+            backends=backends,
+            broadcast_signals_caller=broadcast,
+        )
+    """
+    nodes = {}
+
+    def broadcast_signals_caller(id: str, signals: List[str]):
+        broadcast_signals(id, signals, nodes, backends)
+
+    nodes["router"] = create_router_node_caller(backends, broadcast_signals_caller)
+
+    if call_llm is not None:
+        nodes["llm"] = create_llm_node_caller(backends, call_llm, broadcast_signals_caller)
+
+        tools_list = []
+        if tools_registry:
+            tools_list = [{"function": func, "max_retries": 0} for func in tools_registry.values()]
+        nodes["agent"] = create_agent_node_caller(backends, tools_list, call_llm, broadcast_signals_caller)
+
+    if tools_registry is not None:
+        nodes["tool"] = create_tool_node_caller(backends, tools_registry, broadcast_signals_caller)
+
+    def orchestrate_caller(
+        config: Union[str, Dict[str, Any]],
+        initial_workflow_name: str,
+        initial_signals: List[str],
+        initial_context: Dict[str, Any],
+        backends: Backends,
+    ) -> str:
+        """Start a child workflow execution."""
+        if isinstance(config, str):
+            parsed_config = parse_yaml(config)
+        else:
+            parsed_config = copy.deepcopy(config)
+
+        def child_broadcast(execution_id: str, signals: List[str]):
+            broadcast_signals(execution_id, signals, nodes, backends)
+
+        return orchestrate(
+            config=parsed_config,
+            initial_workflow_name=initial_workflow_name,
+            initial_signals=initial_signals,
+            initial_context=initial_context,
+            backends=backends,
+            broadcast_signals_caller=child_broadcast,
+        )
+
+    nodes["child"] = create_child_node_caller(backends, orchestrate_caller)
+
+    return nodes, broadcast_signals_caller
+
+
+def setup_orchestration(
+    call_llm: Optional[CallLlm] = None,
+    tools_registry: Optional[Dict[str, Callable]] = None,
+    use_local_storage: bool = False,
+    storage_dir: str = "./orchestration_data",
+) -> Tuple[Backends, Callable]:
+    """
+    One-line setup for SOE orchestration.
+
+    Creates backends and nodes with automatic wiring.
+    Returns the backends and broadcast_signals_caller ready to use.
+
+    Args:
+        call_llm: Optional LLM caller function for LLM/Agent nodes
+        tools_registry: Optional dict mapping tool name -> callable
+        use_local_storage: If True, use file-based storage. If False, use in-memory.
+        storage_dir: Directory for local storage (only used if use_local_storage=True)
+
+    Returns:
+        Tuple of (backends, broadcast_signals_caller)
+
+    Example:
+        # Minimal setup (router-only workflows)
+        backends, broadcast = setup_orchestration()
+
+        # Full setup with LLM and tools
+        backends, broadcast = setup_orchestration(
+            call_llm=my_llm_function,
+            tools_registry={"search": search_fn, "calculate": calc_fn},
+        )
+
+        execution_id = orchestrate(
+            config=workflow_yaml,
+            initial_workflow_name="my_workflow",
+            initial_signals=["START"],
+            initial_context={},
+            backends=backends,
+            broadcast_signals_caller=broadcast,
+        )
+    """
+    if use_local_storage:
+        backends = create_local_backends(
+            context_storage_dir=f"{storage_dir}/contexts",
+            workflow_storage_dir=f"{storage_dir}/workflows",
+            telemetry_storage_dir=f"{storage_dir}/telemetry",
+            conversation_history_storage_dir=f"{storage_dir}/conversations",
+            context_schema_storage_dir=f"{storage_dir}/schemas",
+            identity_storage_dir=f"{storage_dir}/identities",
+        )
+    else:
+        backends = create_in_memory_backends()
+
+    _, broadcast_signals_caller = create_all_nodes(
+        backends=backends,
+        call_llm=call_llm,
+        tools_registry=tools_registry,
+    )
+
+    return backends, broadcast_signals_caller
+
+
+__all__ = [
+    "create_all_nodes",
+    "setup_orchestration",
+]

soe/lib/child_context.py

@@ -0,0 +1,46 @@
+"""
+Child context preparation utilities.
+
+Prepares initial context for child workflows with parent metadata.
+"""
+
+import copy
+from typing import Any, Dict
+
+from .context_fields import get_field
+
+
+PARENT_INFO_KEY = "__parent__"
+
+
+def prepare_child_context(
+    parent_context: Dict[str, Any],
+    node_config: Dict[str, Any],
+    parent_execution_id: str,
+    main_execution_id: str,
+) -> Dict[str, Any]:
+    """
+    Prepare initial context for child workflow.
+
+    Includes:
+    - Input fields copied from parent context (current value, not full history)
+    - __parent__ metadata for communication back to parent
+    """
+    child_context: Dict[str, Any] = {}
+
+    # Copy specified input fields from parent (current value only)
+    # orchestrate() will wrap these in history lists
+    for field_name in node_config.get("input_fields", []):
+        if field_name in parent_context:
+            # Use get_field to get current value, not full history
+            child_context[field_name] = copy.deepcopy(get_field(parent_context, field_name))
+
+    # Inject parent info for child-to-parent communication
+    child_context[PARENT_INFO_KEY] = {
+        "parent_execution_id": parent_execution_id,
+        "signals_to_parent": node_config.get("signals_to_parent", []),
+        "context_updates_to_parent": node_config.get("context_updates_to_parent", []),
+        "main_execution_id": main_execution_id,
+    }
+
+    return child_context

soe/lib/context_fields.py

@@ -0,0 +1,51 @@
+"""
+Context field utilities for history-aware field storage.
+
+Fields are stored as lists to maintain update history.
+Reading always returns the last (most recent) value.
+"""
+
+from typing import Any, Dict, List
+
+
+def set_field(context: Dict[str, Any], field: str, value: Any) -> None:
+    """Set a context field, appending to history list."""
+    if field.startswith("__"):
+        context[field] = value
+        return
+
+    if field not in context:
+        context[field] = [value]
+    else:
+        context[field].append(value)
+
+
+def get_field(context: Dict[str, Any], field: str) -> Any:
+    """Get a context field value (last item in history list)."""
+    if field.startswith("__"):
+        return context.get(field)
+
+    value = context.get(field)
+    if value is None:
+        return None
+
+    return value[-1]
+
+
+def get_accumulated(context: Dict[str, Any], field: str) -> List[Any]:
+    """
+    Get full accumulated history for a field.
+
+    If history has exactly one entry and it's a list, returns that list
+    (common case: initial context passed a list as value for fan-out).
+    """
+    if field not in context:
+        return []
+
+    history = context[field]
+
+    # If history has exactly one entry and it's a list, return that list
+    if len(history) == 1 and isinstance(history[0], list):
+        return list(history[0])
+
+    return list(history)

soe/lib/inheritance.py

@@ -0,0 +1,172 @@
+"""
+Configuration and context inheritance utilities.
+
+Handles:
+- Extracting and saving config sections (workflows, identity, schema) from parsed config
+- Inheriting config from existing executions
+- Inheriting context from existing executions
+
+Used for workflow initialization and chaining.
+"""
+
+import copy
+from typing import Dict, Any, Optional
+
+from ..types import Backends
+
+
+def save_config_sections(
+    execution_id: str,
+    backends: Backends,
+    identities: Optional[Dict[str, str]] = None,
+    context_schema: Optional[Dict[str, Any]] = None,
+) -> None:
+    """
+    Save identities and context_schema to their respective backends.
+
+    This is the shared logic for both:
+    - Extracting sections from new config
+    - Inheriting sections from existing execution
+
+    Args:
+        execution_id: Target execution ID
+        backends: Backend services
+        identities: Identity definitions to save (optional)
+        context_schema: Context schema to save (optional)
+    """
+    if identities and backends.identity:
+        backends.identity.save_identities(execution_id, identities)
+
+    if context_schema and backends.context_schema:
+        backends.context_schema.save_context_schema(execution_id, context_schema)
+
+
+def extract_and_save_config_sections(
+    parsed_config: Dict[str, Any],
+    execution_id: str,
+    backends: Backends,
+) -> Dict[str, Any]:
+    """
+    Extract workflows, context_schema, and identities from config.
+
+    If config has 'workflows' key, it's the combined structure:
+    - Extract and save context_schema to context_schema backend
+    - Extract and save identities to identity backend
+    - Return just the workflows portion
+
+    Args:
+        parsed_config: Parsed config dictionary
+        execution_id: Execution ID to save sections under
+        backends: Backend services
+
+    Returns:
+        The workflows registry portion of the config
+    """
+    if "workflows" in parsed_config:
+        workflows = parsed_config["workflows"]
+
+        save_config_sections(
+            execution_id=execution_id,
+            backends=backends,
+            identities=parsed_config.get("identities"),
+            context_schema=parsed_config.get("context_schema"),
+        )
+
+        return workflows
+
+    # Simple/Legacy structure - entire config is workflows
+    return parsed_config
+
+
+def inherit_config(
+    source_execution_id: str,
+    target_execution_id: str,
+    backends: Backends,
+) -> Dict[str, Any]:
+    """
+    Inherit configuration from source execution to target execution.
+
+    Copies:
+    - Workflows registry
+    - Identities (if available)
+    - Context schema (if available)
+
+    Args:
+        source_execution_id: Execution ID to inherit from
+        target_execution_id: Execution ID to inherit to
+        backends: Backend services
+
+    Returns:
+        The workflows registry (for validation and use)
+
+    Raises:
+        ValueError: If source execution has no workflows registry
+    """
+    workflows_registry = backends.workflow.get_workflows_registry(source_execution_id)
+    if not workflows_registry:
+        raise ValueError(
+            f"Cannot inherit config from execution '{source_execution_id}': "
+            "no workflows registry found"
+        )
+
+    # Copy workflows to new execution
+    backends.workflow.save_workflows_registry(target_execution_id, workflows_registry)
+
+    # Get source identities and schema
+    source_identities = None
+    source_schema = None
+
+    if backends.identity:
+        source_identities = backends.identity.get_identities(source_execution_id)
+
+    if backends.context_schema:
+        source_schema = backends.context_schema.get_context_schema(source_execution_id)
+
+    # Save to target using shared logic
+    save_config_sections(
+        execution_id=target_execution_id,
+        backends=backends,
+        identities=source_identities,
+        context_schema=source_schema,
+    )
+
+    return workflows_registry
+
+
+def inherit_context(
+    source_execution_id: str,
+    backends: Backends,
+) -> Dict[str, Any]:
+    """
+    Inherit context from source execution, resetting operational state.
+
+    Copies all context fields except __operational__, which is reset
+    for the new execution.
+
+    Args:
+        source_execution_id: Execution ID to inherit context from
+        backends: Backend services
+
+    Returns:
+        Context dictionary ready for new execution (without __operational__)
+
+    Raises:
+        ValueError: If source execution has no context
+    """
+    source_context = backends.context.get_context(source_execution_id)
+    if not source_context:
+        raise ValueError(
+            f"Cannot inherit context from execution '{source_execution_id}': "
+            "no context found"
+        )
+
+    # Deep copy context, excluding internal fields (will be reset for new execution)
+    # __operational__ - execution tracking state
+    # __parent__ - parent workflow metadata (not relevant for new execution)
+    inherited_context = {
+        k: copy.deepcopy(v)
+        for k, v in source_context.items()
+        if k not in ("__operational__", "__parent__")
+    }
+
+    return inherited_context

soe/lib/jinja_render.py

@@ -0,0 +1,113 @@
+"""
+Jinja template rendering utilities for prompt processing.
+"""
+
+import re
+from typing import Dict, Any, Set, List, Tuple
+
+from jinja2 import Environment, BaseLoader, TemplateSyntaxError
+
+from .context_fields import get_field
+
+
+def _create_accumulated_filter(full_context: Dict[str, Any]):
+    """Create an accumulated filter that returns full history for a field."""
+    def accumulated_filter(value):
+        """
+        Return the full accumulated history list for a context field.
+
+        Usage in templates:
+            {{ context.field | accumulated }}  - returns full list
+            {{ context.field | accumulated | length }}  - count of items
+            {{ context.field | accumulated | join(', ') }}  - join all items
+
+        If history has exactly one entry and it's a list, returns that list
+        (common case: initial context passed a list as value).
+        """
+        # Find the field in full_context by matching the last value
+        for key, hist_list in full_context.items():
+            if key.startswith("__"):
+                continue
+            if isinstance(hist_list, list) and hist_list and hist_list[-1] == value:
+                # If history has exactly one entry and it's a list, return that list
+                if len(hist_list) == 1 and isinstance(hist_list[0], list):
+                    return hist_list[0]
+                return hist_list
+        # Fallback: return value as single-item list
+        return [value] if value is not None else []
+
+    return accumulated_filter
+
+
+def _extract_context_variables(template: str) -> Set[str]:
+    """Extract variable names from a Jinja template."""
+    if not template:
+        return set()
+
+    variables = set()
+
+    dot_pattern = r'\{\{[^}]*context\.([a-zA-Z_][a-zA-Z0-9_]*)'
+    for match in re.finditer(dot_pattern, template):
+        variables.add(match.group(1))
+
+    bracket_pattern = r"\{\{[^}]*context\[['\"]([a-zA-Z_][a-zA-Z0-9_]*)['\"]"
+    for match in re.finditer(bracket_pattern, template):
+        variables.add(match.group(1))
+
+    return variables
+
+
+def get_context_for_prompt(
+    full_context: Dict[str, Any],
+    template: str
+) -> Tuple[Dict[str, Any], List[str]]:
+    """Extract the context needed for a prompt template."""
+    required_fields = _extract_context_variables(template)
+    filtered_context = {}
+    warnings = []
+
+    for field in required_fields:
+        if field not in full_context:
+            warnings.append(f"Context field '{field}' referenced in prompt but not found in context")
+        else:
+            value = get_field(full_context, field)
+            if value is None:
+                warnings.append(f"Context field '{field}' is None")
+                filtered_context[field] = None
+            elif value == "":
+                warnings.append(f"Context field '{field}' is empty string")
+                filtered_context[field] = ""
+            else:
+                filtered_context[field] = value
+
+    return filtered_context, warnings
+
+
+def render_prompt(prompt: str, context: Dict[str, Any]) -> Tuple[str, List[str]]:
+    """Render a Jinja template prompt with the given context."""
+    if not prompt:
+        return prompt, []
+
+    if "{{" not in prompt and "{%" not in prompt:
+        return prompt, []
+
+    _, warnings = get_context_for_prompt(context, prompt)
+
+    unwrapped = {k: get_field(context, k) for k in context if not k.startswith("__")}
+    for k, v in context.items():
+        if k.startswith("__"):
+            unwrapped[k] = v
+
+    try:
+        jinja_env = Environment(loader=BaseLoader())
+        # Register custom filters
+        jinja_env.filters["accumulated"] = _create_accumulated_filter(context)
+        template = jinja_env.from_string(prompt)
+        rendered = template.render(context=unwrapped)
+        return rendered, warnings
+    except TemplateSyntaxError as e:
+        warnings.append(f"Jinja syntax error: {e}")
+        return prompt, warnings
+    except Exception as e:
+        warnings.append(f"Template rendering error: {e}")
+        return prompt, warnings

soe/lib/operational.py

@@ -0,0 +1,51 @@
+"""
+Operational context initialization.
+
+This module handles initialization of the operational state structure.
+Runtime updates are handled by register_event.py.
+"""
+
+from typing import Dict, Any
+
+PARENT_INFO_KEY = "__parent__"
+
+
+def wrap_context_fields(context: Dict[str, Any]) -> Dict[str, Any]:
+    """Wrap context field values in lists for history tracking.
+
+    Internal fields (starting with __) are not wrapped.
+    If context has __parent__, fields are already wrapped (from parent workflow).
+    """
+    # Child workflows receive pre-wrapped fields from parent
+    if PARENT_INFO_KEY in context:
+        return context
+
+    return {
+        k: [v] if not k.startswith("__") else v
+        for k, v in context.items()
+    }
+
+
+def add_operational_state(
+    execution_id: str,
+    context: Dict[str, Any],
+) -> Dict[str, Any]:
+    """Add operational state to context if not present. Returns new context dict."""
+    if "__operational__" in context:
+        return context
+
+    parent_info = context.get(PARENT_INFO_KEY, {})
+    inherited_main_id = parent_info.get("main_execution_id")
+    main_id = inherited_main_id if inherited_main_id else execution_id
+
+    return {
+        **context,
+        "__operational__": {
+            "signals": [],
+            "nodes": {},
+            "llm_calls": 0,
+            "tool_calls": 0,
+            "errors": 0,
+            "main_execution_id": main_id
+        }
+    }

soe/lib/parent_sync.py

@@ -0,0 +1,71 @@
+"""
+Parent sync utilities for sub-orchestration.
+
+These functions check if a signal or context update should be propagated
+to the parent workflow based on the injected __parent__ metadata.
+"""
+
+from typing import Dict, Any, Tuple, Optional, List
+from ..types import Backends
+
+PARENT_INFO_KEY = "__parent__"
+
+
+def get_signals_for_parent(
+    signals: List[str], context: Dict[str, Any]
+) -> Tuple[Optional[str], List[str]]:
+    """Get the subset of signals that should be propagated to the parent."""
+    parent_info = context.get(PARENT_INFO_KEY)
+    if not parent_info:
+        return (None, [])
+
+    parent_execution_id = parent_info.get("parent_execution_id")
+    signals_to_parent = set(parent_info.get("signals_to_parent", []))
+    matching_signals = [s for s in signals if s in signals_to_parent]
+
+    return (parent_execution_id, matching_signals)
+
+
+def _check_parent_context_sync(
+    key: str, context: Dict[str, Any]
+) -> Tuple[Optional[str], bool]:
+    """Determine if a context update should be propagated to the parent."""
+    parent_info = context.get(PARENT_INFO_KEY)
+    if not parent_info:
+        return (None, False)
+
+    parent_execution_id = parent_info.get("parent_execution_id")
+    context_updates_to_parent = parent_info.get("context_updates_to_parent", [])
+
+    if key in context_updates_to_parent:
+        return (parent_execution_id, True)
+
+    return (parent_execution_id, False)
+
+
+def sync_context_to_parent(
+    context: Dict[str, Any],
+    updated_keys: List[str],
+    backends: Backends,
+) -> None:
+    """Sync updated context keys to the parent workflow if configured.
+
+    For each key in updated_keys that is configured for parent sync:
+    - If parent doesn't have the key, copy the full list from child
+    - If parent already has the key, extend with new items from child
+    """
+    for key in updated_keys:
+        parent_id, should_sync = _check_parent_context_sync(key, context)
+        if should_sync and parent_id:
+            parent_context = backends.context.get_context(parent_id)
+            child_history = context[key]
+
+            if key in parent_context:
+                # Append new items from child to parent's existing list
+                parent_context[key].extend(child_history)
+            else:
+                # Initialize parent with child's history
+                parent_context[key] = child_history
+
+            backends.context.save_context(parent_id, parent_context)
+            sync_context_to_parent(parent_context, [key], backends)