soe-ai 0.2.0b1__py3-none-any.whl

soe/validation/config.py

@@ -0,0 +1,195 @@
+"""
+Config validation - validates config structure before orchestration starts.
+
+Supports two config formats:
+1. Single Workflow format: Dict of workflow definitions directly
+2. Combined config format: Dict with 'workflows', 'context_schema', 'identities' keys
+
+Runs once at orchestration start, before any execution.
+"""
+
+from typing import Dict, Any
+
+from ..types import WorkflowValidationError
+from ..nodes.router.validation import validate_node_config as validate_router
+from ..nodes.agent.validation import validate_node_config as validate_agent
+from ..nodes.llm.validation import validate_node_config as validate_llm
+from ..nodes.child.validation import validate_node_config as validate_child
+from ..nodes.tool.validation import validate_node_config as validate_tool
+from ..lib.yaml_parser import parse_yaml
+
+
+NODE_VALIDATORS = {
+    "router": validate_router,
+    "agent": validate_agent,
+    "llm": validate_llm,
+    "child": validate_child,
+    "tool": validate_tool,
+}
+
+
+def _validate_workflow_section(workflow_name: str, workflow: Dict[str, Any]) -> None:
+    """
+    Validate all nodes in a workflow section.
+
+    Args:
+        workflow_name: Name of the workflow (for error messages)
+        workflow: Workflow definition dict
+
+    Raises:
+        WorkflowValidationError: If any node configuration is invalid
+    """
+    if not workflow:
+        raise WorkflowValidationError(
+            f"Workflow '{workflow_name}' is empty - at least one node is required"
+        )
+
+    for node_name, node_config in workflow.items():
+        if node_name.startswith("__"):
+            raise WorkflowValidationError(
+                f"Workflow '{workflow_name}': node name '{node_name}' is reserved - "
+                f"names starting with '__' are reserved for internal use"
+            )
+        node_type = node_config.get("node_type")
+
+        if not node_type:
+            raise WorkflowValidationError(
+                f"Workflow '{workflow_name}', node '{node_name}': "
+                f"'node_type' is required - specify the type (router, agent, llm, tool, child)"
+            )
+
+        if node_type.startswith("_"):
+            continue
+
+        validator = NODE_VALIDATORS.get(node_type)
+        if not validator:
+            valid_types = ", ".join(NODE_VALIDATORS.keys())
+            raise WorkflowValidationError(
+                f"Workflow '{workflow_name}', node '{node_name}': "
+                f"unknown node_type '{node_type}'. Valid types are: {valid_types}"
+            )
+
+        try:
+            validator(node_config)
+        except WorkflowValidationError as e:
+            raise WorkflowValidationError(
+                f"Workflow '{workflow_name}', node '{node_name}': {e}"
+            ) from e
+
+
+def _validate_context_schema_section(context_schema: Dict[str, Any]) -> None:
+    """
+    Validate context_schema section of config.
+
+    Args:
+        context_schema: Context schema definitions
+
+    Raises:
+        WorkflowValidationError: If schema format is invalid
+    """
+    if not isinstance(context_schema, dict):
+        raise WorkflowValidationError(
+            "'context_schema' section must be an object mapping field names to schemas"
+        )
+
+    for field_name, field_schema in context_schema.items():
+        if not isinstance(field_schema, (dict, str)):
+            raise WorkflowValidationError(
+                f"context_schema.{field_name}: schema must be an object or type string"
+            )
+
+
+def _validate_identities_section(identities: Dict[str, str]) -> None:
+    """
+    Validate identities section of config.
+
+    Args:
+        identities: Identity definitions (name -> system prompt)
+
+    Raises:
+        WorkflowValidationError: If identity format is invalid
+    """
+    if not isinstance(identities, dict):
+        raise WorkflowValidationError(
+            "'identities' section must be an object mapping identity names to prompts"
+        )
+
+    for identity_name, identity_prompt in identities.items():
+        if not isinstance(identity_prompt, str):
+            raise WorkflowValidationError(
+                f"identities.{identity_name}: identity prompt must be a string, "
+                f"got {type(identity_prompt).__name__}"
+            )
+
+
+def validate_config(config) -> Dict[str, Any]:
+    """
+    Parse and validate config.
+
+    Supports two formats:
+    1. Legacy format: Dict of workflow definitions directly
+    2. Combined config format: Dict with 'workflows', 'context_schema', 'identities' keys
+
+    Args:
+        config: YAML string or dict (workflows only or combined config)
+
+    Returns:
+        Parsed config dict (either workflows directly or combined structure)
+
+    Raises:
+        WorkflowValidationError: If any configuration is invalid
+    """
+    parsed = parse_yaml(config)
+
+    if "workflows" in parsed:
+        workflows = parsed["workflows"]
+        if not isinstance(workflows, dict):
+            raise WorkflowValidationError(
+                "'workflows' section must be an object containing workflow definitions"
+            )
+        for workflow_name, workflow in workflows.items():
+            if not isinstance(workflow, dict):
+                raise WorkflowValidationError(
+                    f"Workflow '{workflow_name}' must be an object containing node definitions"
+                )
+            _validate_workflow_section(workflow_name, workflow)
+
+        context_schema = parsed.get("context_schema")
+        if context_schema is not None:
+            _validate_context_schema_section(context_schema)
+
+        identities = parsed.get("identities")
+        if identities is not None:
+            _validate_identities_section(identities)
+    else:
+        for workflow_name, workflow in parsed.items():
+            if not isinstance(workflow, dict):
+                raise WorkflowValidationError(
+                    f"Workflow '{workflow_name}' must be an object containing node definitions"
+                )
+            _validate_workflow_section(workflow_name, workflow)
+
+    return parsed
+
+
+validate_workflow = _validate_workflow_section
+
+
+def validate_orchestrate_params(
+    initial_workflow_name: str,
+    initial_signals: list,
+) -> None:
+    """Validate orchestrate() parameters before execution starts."""
+    if not isinstance(initial_signals, list):
+        raise WorkflowValidationError(
+            f"'initial_signals' must be a list, got {type(initial_signals).__name__}. "
+            f"Example: initial_signals=['START']"
+        )
+    if not initial_signals:
+        raise WorkflowValidationError(
+            "'initial_signals' cannot be empty - at least one signal is required to start execution"
+        )
+    if not isinstance(initial_workflow_name, str) or not initial_workflow_name:
+        raise WorkflowValidationError(
+            "'initial_workflow_name' must be a non-empty string"
+        )

soe/validation/jinja.py

@@ -0,0 +1,54 @@
+"""
+Jinja template validation utilities.
+
+Used during config validation to catch Jinja errors early.
+"""
+
+from jinja2 import Environment, BaseLoader, TemplateSyntaxError
+from ..types import WorkflowValidationError
+
+
+def _dummy_accumulated_filter(value):
+    """Dummy accumulated filter for validation - just returns value as list."""
+    return [value] if value is not None else []
+
+
+def validate_jinja_syntax(template: str, context_description: str) -> None:
+    """
+    Validate Jinja template syntax at config time.
+
+    Called during config validation to catch Jinja errors early.
+
+    Catches:
+    - Unclosed braces {{ without }}
+    - Unknown filters like | capitalize_all
+    - Basic syntax errors
+
+    Does NOT catch:
+    - Runtime errors like division by zero (depends on context values)
+    - Undefined variables (depends on context at runtime)
+
+    Args:
+        template: The Jinja template string to validate
+        context_description: Description for error messages (e.g., "condition for signal 'DONE'")
+
+    Raises:
+        WorkflowValidationError: If template has syntax or filter errors
+    """
+    if not template or ("{{" not in template and "{%" not in template):
+        return
+    try:
+        env = Environment(loader=BaseLoader())
+        env.filters["accumulated"] = _dummy_accumulated_filter
+        env.parse(template)
+        env.from_string(template)
+    except TemplateSyntaxError as e:
+        raise WorkflowValidationError(
+            f"{context_description}: Jinja syntax error - {e.message}"
+        )
+    except Exception as e:
+        error_msg = str(e)
+        if "filter" in error_msg.lower():
+            raise WorkflowValidationError(
+                f"{context_description}: {error_msg}"
+            )

soe/validation/operational.py

@@ -0,0 +1,110 @@
+"""
+Operational validation - validates runtime state before node execution.
+
+This validates that the context and backends are in a valid state
+for node execution. Runs before each node executes so that operational
+code can trust the structure and avoid defensive programming.
+
+Fail-fast: If validation fails, raise immediately with clear message.
+"""
+
+from typing import Dict, Any
+
+from ..types import Backends, SoeError
+
+
+class OperationalValidationError(SoeError):
+    """Raised when operational context is invalid."""
+    pass
+
+
+def validate_operational(
+    execution_id: str,
+    backends: Backends,
+) -> Dict[str, Any]:
+    """
+    Validate that operational context exists and has required structure.
+
+    Call this before any node execution to ensure __operational__ is valid.
+    Returns the context so caller doesn't need to fetch it again.
+
+    Args:
+        execution_id: The execution ID
+        backends: Backend services
+
+    Returns:
+        The validated context dict
+
+    Raises:
+        OperationalValidationError: If context or __operational__ is invalid
+    """
+    context = backends.context.get_context(execution_id)
+
+    if not context:
+        raise OperationalValidationError(
+            f"No context found for execution_id '{execution_id}'. "
+            f"Context must be initialized before node execution."
+        )
+
+    operational = context.get("__operational__")
+
+    if operational is None:
+        raise OperationalValidationError(
+            f"Missing '__operational__' in context for execution_id '{execution_id}'. "
+            f"Call initialize_operational_context() before node execution."
+        )
+
+    required_fields = ["signals", "nodes", "llm_calls", "tool_calls", "errors", "main_execution_id"]
+    missing = [f for f in required_fields if f not in operational]
+
+    if missing:
+        raise OperationalValidationError(
+            f"Invalid '__operational__' structure for execution_id '{execution_id}'. "
+            f"Missing fields: {missing}"
+        )
+
+    if not isinstance(operational["signals"], list):
+        raise OperationalValidationError(
+            f"Invalid '__operational__.signals' - must be a list, got {type(operational['signals']).__name__}"
+        )
+
+    if not isinstance(operational["nodes"], dict):
+        raise OperationalValidationError(
+            f"Invalid '__operational__.nodes' - must be a dict, got {type(operational['nodes']).__name__}"
+        )
+
+    if not isinstance(operational["llm_calls"], int):
+        raise OperationalValidationError(
+            f"Invalid '__operational__.llm_calls' - must be an int, got {type(operational['llm_calls']).__name__}"
+        )
+
+    if not isinstance(operational["tool_calls"], int):
+        raise OperationalValidationError(
+            f"Invalid '__operational__.tool_calls' - must be an int, got {type(operational['tool_calls']).__name__}"
+        )
+
+    if not isinstance(operational["errors"], int):
+        raise OperationalValidationError(
+            f"Invalid '__operational__.errors' - must be an int, got {type(operational['errors']).__name__}"
+        )
+
+    return context
+
+
+def validate_backends(backends: Backends) -> None:
+    """
+    Validate that backends has required attributes.
+
+    Args:
+        backends: Backend services
+
+    Raises:
+        OperationalValidationError: If backends is invalid
+    """
+    required = ["context", "workflow"]
+
+    for attr in required:
+        if not hasattr(backends, attr) or getattr(backends, attr) is None:
+            raise OperationalValidationError(
+                f"Invalid backends: missing required attribute '{attr}'"
+            )

soe_ai-0.2.0b1.dist-info/METADATA

@@ -0,0 +1,262 @@
+Metadata-Version: 2.4
+Name: soe-ai
+Version: 0.2.0b1
+Summary: Signal-driven Orchestration Engine - Agent orchestration with event-driven workflow engine
+Author-email: Pedro Garcia <pgarcia14180@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/pgarcia14180/soe
+Project-URL: Documentation, https://github.com/pgarcia14180/soe/tree/master/docs
+Project-URL: Repository, https://github.com/pgarcia14180/soe
+Project-URL: Issues, https://github.com/pgarcia14180/soe/issues
+Keywords: orchestration,agent,workflow,automation
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: jinja2>=2.11.3
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: black>=23.0; extra == "dev"
+Requires-Dist: mypy>=1.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Provides-Extra: integration
+Requires-Dist: openai>=1.0.0; extra == "integration"
+Requires-Dist: requests>=2.28.0; extra == "integration"
+Dynamic: license-file
+
+# SOE — Signal-driven Orchestration Engine
+
+**A protocol for orchestrating AI workflows through signals.**
+
+---
+
+## What SOE Is
+
+SOE is an orchestration engine where nodes communicate through **signals** rather than direct function calls. You define workflows in YAML, and the engine handles execution.
+
+| Approach | How It Works | Trade-off |
+|----------|--------------|-----------|
+| Chain-based | `Step A → B → C → D` | Simple but rigid |
+| SOE Signal-based | `[SIGNAL] → all listeners respond` | Flexible, requires understanding signals |
+
+**The C++ analogy**: Like C++ gives you control over memory and execution (compared to higher-level languages), SOE gives you control over orchestration primitives. You decide how state is stored, how LLMs are called, and how signals are broadcast. This requires more setup but means no vendor lock-in and full observability.
+
+---
+
+## What SOE Does
+
+SOE orchestrates workflows through **signals**. Nodes don't call each other—they emit signals that other nodes listen for.
+
+```yaml
+example_workflow:
+  ValidateInput:
+    node_type: router
+    event_triggers: [START]
+    event_emissions:
+      - signal_name: VALID
+        condition: "{{ context.data is defined }}"
+      - signal_name: INVALID
+
+  ProcessData:
+    node_type: llm
+    event_triggers: [VALID]
+    prompt: "Process this: {{ context.data }}"
+    output_field: result
+    event_emissions:
+      - signal_name: DONE
+```
+
+**That's the entire workflow definition.** No SDK, no decorators, no base classes.
+
+---
+
+## Why SOE
+
+### 1. Infrastructure-Agnostic
+SOE defines **protocols**, not implementations. Swap PostgreSQL for DynamoDB. Replace OpenAI with a local LLM. Deploy to Lambda, Kubernetes, or a single Python script. Your workflow YAML stays the same.
+
+### 2. Context-Driven with Jinja
+All workflow state flows through **context**—a shared dictionary accessible via Jinja2 templates. This means:
+- Conditions like `{{ context.user_validated }}` are readable and debuggable
+- LLM prompts can interpolate any context field
+- No hidden state—everything is inspectable
+
+### 3. Purely Deterministic or Hybrid Agentic
+SOE is a complete orchestration solution. You can use it as a purely deterministic engine for standard business logic, or mix in LLM-driven "Agentic" behavior.
+- **Deterministic**: Use `router` and `tool` nodes for 100% predictable workflows.
+- **Agentic**: Add `llm` and `agent` nodes for creative, reasoning-based tasks.
+You get the safety of code with the flexibility of AI in a single, unified system.
+
+### 4. Portable
+Workflows are YAML. Run them locally, in CI, in production. Extract them, version them, share them.
+
+### 5. Self-Evolving
+Workflows can modify themselves at runtime. Built-in tools like `inject_workflow`, `inject_node_configuration`, and `add_signal` allow agents to:
+- Create new workflows dynamically
+- Add or modify nodes in existing workflows
+- Update signal routing on the fly
+
+This enables **meta-programming**: an AI system that can extend its own capabilities without human intervention.
+
+---
+
+## What SOE Unlocks
+
+SOE is a **Protocol for Intelligence** that unlocks new forms of intelligent behavior:
+
+### Self-Evolving Intelligence
+AI systems that can rewrite and improve themselves at runtime - the ultimate evolution of software.
+
+### Swarm Intelligence
+Efficient collective decision-making among multiple agents through signal-based consensus.
+
+### Hybrid Intelligence
+Seamless combination of deterministic logic and AI creativity with programmatic safety rails.
+
+### Fractal Intelligence
+Hierarchical agent organizations that scale complexity while remaining manageable.
+
+### Infrastructure Intelligence
+AI orchestration that works everywhere - from edge devices to cloud platforms.
+
+---
+
+## Installation
+
+```bash
+# With uv (recommended)
+uv add soe-ai
+
+# With pip
+pip install soe-ai
+
+# From source
+git clone https://github.com/pgarcia14180/soe.git
+cd soe && uv sync
+```
+
+---
+
+## Quick Start
+
+### 1. Provide Your LLM
+
+SOE is LLM-agnostic. You must provide a `call_llm` function that matches this signature:
+
+```python
+def call_llm(
+    prompt: str,
+    config: dict,
+) -> str:
+    """
+    Called by SOE when a node needs LLM processing.
+
+    Args:
+        prompt: The rendered prompt string (includes instructions, context, and schemas)
+        config: The full node configuration from YAML (useful for model parameters)
+
+    Returns:
+        The raw text response from the LLM.
+    """
+    # Example with OpenAI:
+    from openai import OpenAI
+    client = OpenAI()
+    response = client.chat.completions.create(
+        model=config.get("model", "gpt-4o"),
+        messages=[{"role": "user", "content": prompt}],
+    )
+    return response.choices[0].message.content
+```
+
+### 2. Run a Workflow
+
+```python
+from soe import orchestrate, create_all_nodes
+from soe.local_backends import create_local_backends
+
+# Your workflow (can also be loaded from file or database)
+workflow = """
+example_workflow:
+  Start:
+    node_type: router
+    event_triggers: [START]
+    event_emissions:
+      - signal_name: DONE
+"""
+
+# Create backends (storage for context, workflows, etc.)
+backends = create_local_backends("./data")
+
+# Create all node handlers (pass your call_llm function)
+nodes, broadcast = create_all_nodes(backends, call_llm=call_llm)
+
+# Run the workflow
+execution_id = orchestrate(
+    config=workflow,
+    initial_workflow_name="example_workflow",
+    initial_signals=["START"],
+    initial_context={"user": "alice"},
+    backends=backends,
+    broadcast_signals_caller=broadcast,
+)
+# When orchestrate() returns, the workflow is complete
+```
+
+**For product managers and less technical users**: The Quick Start above is all you need to run a workflow. The `config` parameter accepts YAML defining your workflow structure. The `initial_context` is where you pass input data (like user IDs, requests, etc.).
+
+---
+
+## Documentation
+
+| Audience | Start Here |
+|----------|------------|
+| **Builders** (workflow authors) | [Documentation](soe/docs/index.md) — Step-by-step chapters |
+| **Engineers** (infrastructure) | [Infrastructure Guide](soe/docs/guide_10_infrastructure.md) — Backend protocols |
+| **Researchers** (advanced patterns) | [Advanced Patterns](docs/advanced_patterns/) — Swarm, hybrid, self-evolving |
+
+---
+
+## Node Types
+
+| Node | Purpose |
+|------|---------|
+| `router` | Conditional signal emission (no LLM) |
+| `llm` | Single LLM call with output |
+| `agent` | Multi-turn LLM with tool access |
+| `tool` | Execute Python functions |
+| `child` | Spawn sub-workflows |
+
+---
+
+## Backend Protocols
+
+Implement these to plug SOE into your infrastructure:
+
+| Protocol | Purpose |
+|----------|---------|
+| `ContextBackend` | Workflow state storage |
+| `WorkflowBackend` | Workflow definitions |
+| `ContextSchemaBackend` | Output validation (optional) |
+| `IdentityBackend` | LLM system prompts (optional) |
+| `ConversationHistoryBackend` | Agent memory (optional) |
+| `TelemetryBackend` | Observability (optional) |
+
+**Recommendation**: Use the same database for context, workflows, identities, and context_schema—just separate tables. The backend methods handle table creation.
+
+See [Infrastructure Guide](docs/guide_10_infrastructure.md) for PostgreSQL, DynamoDB, and Lambda examples.
+
+---
+
+## License
+
+MIT