evalforge-runtime 0.1.0__py3-none-any.whl

evalforge_runtime/__init__.py

@@ -0,0 +1,37 @@
+"""EvalForge Runtime — execution engine for generated applications."""
+
+__version__ = "0.1.0"
+
+
+def main() -> None:
+    """CLI entry point: evalforge-runtime start --config <path>"""
+    import argparse
+
+    from evalforge_runtime.config import load_config
+    from evalforge_runtime.server import create_app
+
+    parser = argparse.ArgumentParser(
+        prog="evalforge-runtime",
+        description="EvalForge Runtime Server",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    start_parser = subparsers.add_parser("start", help="Start the runtime server")
+    start_parser.add_argument(
+        "--config", default="evalforge.config.yaml", help="Path to config YAML"
+    )
+    start_parser.add_argument("--host", default="0.0.0.0", help="Bind host")
+    start_parser.add_argument("--port", type=int, default=8000, help="Bind port")
+
+    args = parser.parse_args()
+
+    config_path = getattr(args, "config", "evalforge.config.yaml")
+    host = getattr(args, "host", "0.0.0.0")
+    port = getattr(args, "port", 8000)
+
+    config = load_config(config_path)
+    app = create_app(config)
+
+    import uvicorn
+
+    uvicorn.run(app, host=host, port=port)

evalforge_runtime/__main__.py

@@ -0,0 +1,5 @@
+"""Enable python -m evalforge_runtime."""
+
+from evalforge_runtime import main
+
+main()

evalforge_runtime/auth.py

@@ -0,0 +1,33 @@
+"""API key authentication middleware."""
+
+from __future__ import annotations
+
+import os
+
+from fastapi import HTTPException, Request
+
+from evalforge_runtime.config import AuthConfig
+
+
+class APIKeyAuth:
+    """FastAPI dependency that validates API key authentication."""
+
+    def __init__(self, config: AuthConfig):
+        self.config = config
+        self.api_key_methods = [m for m in config.methods if m.type == "api_key"]
+
+    async def __call__(self, request: Request) -> str:
+        """Validate the request. Returns the authenticated key."""
+        for method in self.api_key_methods:
+            header_name = method.header or "X-API-Key"
+            key = request.headers.get(header_name)
+            if key and self._validate_key(key):
+                return key
+
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+    def _validate_key(self, key: str) -> bool:
+        """Check if the key is in the allowed list."""
+        valid_keys = os.environ.get("EVALFORGE_API_KEYS", "").split(",")
+        valid_keys = [k.strip() for k in valid_keys if k.strip()]
+        return key in valid_keys

evalforge_runtime/config.py

@@ -0,0 +1,157 @@
+"""Pydantic models for evalforge.config.yaml and config loading."""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel
+
+
+class ProjectConfig(BaseModel):
+    id: str
+    evalforge_url: str | None = None
+    version: str = "0.0.0"
+
+
+class SecretConfig(BaseModel):
+    provider: Literal[
+        "evalforge", "env", "azure_keyvault", "aws_secrets_manager", "sap_credential_store"
+    ] = "env"
+    # Azure Key Vault
+    vault_url: str | None = None
+    # AWS Secrets Manager
+    region: str | None = None
+    secret_name: str | None = None
+    # SAP Credential Store
+    instance: str | None = None
+
+
+class AuthMethod(BaseModel):
+    type: Literal["oauth2", "api_key"]
+    issuer: str | None = None
+    audience: str | None = None
+    header: str | None = None
+
+
+class AuthConfig(BaseModel):
+    methods: list[AuthMethod] = []
+
+
+class LLMConfig(BaseModel):
+    model: str = "anthropic/claude-sonnet-4-6"
+
+
+class ReviewConfig(BaseModel):
+    enabled: bool = False
+    timeout: str = "24h"
+
+
+class LangfuseConfig(BaseModel):
+    enabled: bool = False
+    host: str | None = None
+
+
+class ObservabilityConfig(BaseModel):
+    langfuse: LangfuseConfig = LangfuseConfig()
+
+
+class TriggerConfig(BaseModel):
+    type: Literal["schedule", "webhook", "process"]
+    cron: str | None = None
+    after: str | None = None
+
+
+class ProcessConfig(BaseModel):
+    process_id: str
+    instructions: str = ""
+    trigger: TriggerConfig
+    connector: str | None = None
+    connector_params: dict[str, Any] | None = None
+    review: ReviewConfig = ReviewConfig()
+    llm_model: str | None = None
+    output_schema: dict[str, str] | None = None
+    before_module: str | None = None
+    execution_module: str | None = None
+    after_module: str | None = None
+
+
+class DatabaseConfig(BaseModel):
+    url: str = "sqlite+aiosqlite:///./data/app.db"
+
+
+class StorageConfig(BaseModel):
+    type: Literal["local", "s3"] = "local"
+    path: str = "./data/files"
+    bucket: str | None = None
+    region: str | None = None
+
+
+class AppConfig(BaseModel):
+    project: ProjectConfig
+    secrets: SecretConfig = SecretConfig()
+    auth: AuthConfig = AuthConfig()
+    llm: LLMConfig = LLMConfig()
+    database: DatabaseConfig = DatabaseConfig()
+    storage: StorageConfig = StorageConfig()
+    observability: ObservabilityConfig = ObservabilityConfig()
+    processes: dict[str, ProcessConfig] = {}
+
+
+def _parse_duration(duration_str: str) -> int:
+    """Parse duration string like '24h', '30m', '7d' to seconds."""
+    duration_str = duration_str.strip().lower()
+    if duration_str.endswith("d"):
+        return int(duration_str[:-1]) * 86400
+    elif duration_str.endswith("h"):
+        return int(duration_str[:-1]) * 3600
+    elif duration_str.endswith("m"):
+        return int(duration_str[:-1]) * 60
+    elif duration_str.endswith("s"):
+        return int(duration_str[:-1])
+    else:
+        return int(duration_str)
+
+
+_ENV_VAR_PATTERN = re.compile(r"\$\{([^}]+)}")
+
+
+def _resolve_env_vars(value: Any) -> Any:
+    """Recursively resolve ${ENV_VAR} patterns in config values."""
+    if isinstance(value, str):
+        def replacer(match: re.Match[str]) -> str:
+            var_name = match.group(1)
+            env_value = os.environ.get(var_name)
+            if env_value is None:
+                return match.group(0)  # leave unresolved
+            return env_value
+
+        return _ENV_VAR_PATTERN.sub(replacer, value)
+    elif isinstance(value, dict):
+        return {k: _resolve_env_vars(v) for k, v in value.items()}
+    elif isinstance(value, list):
+        return [_resolve_env_vars(item) for item in value]
+    return value
+
+
+def load_config(path: str | Path) -> AppConfig:
+    """Load and validate an evalforge.config.yaml file."""
+    from dotenv import load_dotenv
+
+    load_dotenv()
+
+    config_path = Path(path)
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+
+    with open(config_path) as f:
+        raw = yaml.safe_load(f)
+
+    if not isinstance(raw, dict):
+        raise ValueError(f"Config file must contain a YAML mapping, got {type(raw).__name__}")
+
+    resolved = _resolve_env_vars(raw)
+    return AppConfig.model_validate(resolved)

evalforge_runtime/connectors/__init__.py

@@ -0,0 +1,5 @@
+"""Connectors provide input (triggers/sources) and output (actions) for processes."""
+
+from evalforge_runtime.connectors.base import Connector
+
+__all__ = ["Connector"]

evalforge_runtime/connectors/base.py

@@ -0,0 +1,40 @@
+"""Connector base class."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from evalforge_runtime.types import FileRef
+
+
+@dataclass
+class ConnectorItem:
+    """A single item fetched by a connector (e.g., one email)."""
+
+    ref: str  # unique identifier (e.g., email message ID)
+    data: dict[str, Any]  # the item's data fields
+    attachments: list[FileRef] = field(default_factory=list)
+
+
+class Connector(ABC):
+    """Base class for all connectors."""
+
+    def __init__(self, params: dict[str, Any] | None = None, secrets: dict[str, str] | None = None):
+        self.params = params or {}
+        self.secrets = secrets or {}
+
+    @abstractmethod
+    async def fetch(self) -> list[ConnectorItem]:
+        """Fetch new items from the source. Called on each trigger."""
+        ...
+
+    async def validate(self) -> None:
+        """Validate credentials and config on startup. Raises on failure."""
+        pass
+
+    @abstractmethod
+    def name(self) -> str:
+        """Return the connector name (e.g., 'exchange-inbox')."""
+        ...

evalforge_runtime/connectors/exchange.py

@@ -0,0 +1,255 @@
+"""Microsoft Exchange Online connector via Microsoft Graph API.
+
+Supports shared mailboxes — the service principal needs
+Mail.ReadWrite application permission on the target mailbox.
+
+Required secrets:
+  EXCHANGE_TENANT_ID
+  EXCHANGE_CLIENT_ID
+  EXCHANGE_CLIENT_SECRET
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+from evalforge_runtime.connectors.base import Connector, ConnectorItem
+from evalforge_runtime.storage import LocalStorage
+from evalforge_runtime.types import FileRef
+
+logger = logging.getLogger(__name__)
+
+
+class ExchangeConnector(Connector):
+    """Microsoft Exchange connector via Graph API."""
+
+    def __init__(
+        self,
+        params: dict[str, Any] | None = None,
+        secrets: dict[str, str] | None = None,
+        storage: LocalStorage | None = None,
+    ):
+        super().__init__(params, secrets)
+        self.storage = storage
+        self._token: str | None = None
+        self._mailbox = self.params.get("mailbox", "")
+        self._folder = self.params.get("folder", "Inbox")
+        self._filter = self.params.get("filter", "unread")
+
+    def name(self) -> str:
+        return "exchange-inbox"
+
+    async def validate(self) -> None:
+        """Validate Exchange credentials by acquiring a token."""
+        required = ["EXCHANGE_TENANT_ID", "EXCHANGE_CLIENT_ID", "EXCHANGE_CLIENT_SECRET"]
+        missing = [k for k in required if k not in self.secrets]
+        if missing:
+            raise ValueError(f"Exchange connector missing secrets: {', '.join(missing)}")
+        if not self._mailbox:
+            raise ValueError("Exchange connector requires 'mailbox' parameter")
+        await self._acquire_token()
+
+    async def _acquire_token(self) -> str:
+        """Acquire OAuth2 token via client credentials flow."""
+        if self._token:
+            return self._token
+
+        tenant_id = self.secrets["EXCHANGE_TENANT_ID"]
+        async with httpx.AsyncClient() as client:
+            resp = await client.post(
+                f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token",
+                data={
+                    "grant_type": "client_credentials",
+                    "client_id": self.secrets["EXCHANGE_CLIENT_ID"],
+                    "client_secret": self.secrets["EXCHANGE_CLIENT_SECRET"],
+                    "scope": "https://graph.microsoft.com/.default",
+                },
+            )
+            resp.raise_for_status()
+            self._token = resp.json()["access_token"]
+            return self._token
+
+    async def fetch(self) -> list[ConnectorItem]:
+        """Fetch unread messages from the configured mailbox."""
+        token = await self._acquire_token()
+        headers = {"Authorization": f"Bearer {token}"}
+
+        filter_str = "$filter=isRead eq false" if self._filter == "unread" else ""
+        url = (
+            f"https://graph.microsoft.com/v1.0/users/{self._mailbox}"
+            f"/mailFolders/{self._folder}/messages?{filter_str}&$top=50"
+        )
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url, headers=headers)
+            resp.raise_for_status()
+            messages = resp.json().get("value", [])
+
+        items: list[ConnectorItem] = []
+        for msg in messages:
+            attachments: list[FileRef] = []
+            if msg.get("hasAttachments") and self.storage:
+                attachments = await self._fetch_attachments(
+                    msg["id"], headers, msg["id"]
+                )
+
+            items.append(ConnectorItem(
+                ref=msg["id"],
+                data={
+                    "email_subject": msg.get("subject", ""),
+                    "email_body": msg.get("body", {}).get("content", ""),
+                    "sender": msg.get("from", {}).get("emailAddress", {}).get("address", ""),
+                    "received_at": msg.get("receivedDateTime", ""),
+                },
+                attachments=attachments,
+            ))
+
+        return items
+
+    async def _fetch_attachments(
+        self, message_id: str, headers: dict, execution_ref: str
+    ) -> list[FileRef]:
+        """Fetch and store attachments for a message."""
+        if not self.storage:
+            return []
+
+        url = (
+            f"https://graph.microsoft.com/v1.0/users/{self._mailbox}"
+            f"/messages/{message_id}/attachments"
+        )
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url, headers=headers)
+            resp.raise_for_status()
+            attachments_data = resp.json().get("value", [])
+
+        refs: list[FileRef] = []
+        for att in attachments_data:
+            if att.get("@odata.type") != "#microsoft.graph.fileAttachment":
+                continue
+
+            import base64
+            content = base64.b64decode(att.get("contentBytes", ""))
+            filename = att.get("name", "attachment")
+            extension = filename.rsplit(".", 1)[-1] if "." in filename else ""
+            mime_type = att.get("contentType", "application/octet-stream")
+            key = f"connector/exchange/{execution_ref}/{filename}"
+
+            await self.storage.put(key, content, mime_type)
+
+            refs.append(FileRef(
+                type="local",
+                key=key,
+                filename=filename,
+                size=len(content),
+                mimeType=mime_type,
+                extension=extension,
+            ))
+
+        return refs
+
+    # --- Output methods (used by after steps) ---
+
+    async def send_message(
+        self, to: list[str], subject: str, body: str,
+        attachments: list[FileRef] | None = None,
+    ) -> str:
+        """Send a new email. Returns message ID."""
+        token = await self._acquire_token()
+        message: dict[str, Any] = {
+            "subject": subject,
+            "body": {"contentType": "HTML", "content": body},
+            "toRecipients": [{"emailAddress": {"address": addr}} for addr in to],
+        }
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.post(
+                f"https://graph.microsoft.com/v1.0/users/{self._mailbox}/sendMail",
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                json={"message": message, "saveToSentItems": True},
+            )
+            resp.raise_for_status()
+        return "sent"
+
+    async def reply(self, message_id: str, body: str, reply_all: bool = False) -> None:
+        """Reply to an existing message."""
+        token = await self._acquire_token()
+        endpoint = "replyAll" if reply_all else "reply"
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.post(
+                f"https://graph.microsoft.com/v1.0/users/{self._mailbox}"
+                f"/messages/{message_id}/{endpoint}",
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                json={"comment": body},
+            )
+            resp.raise_for_status()
+
+    async def forward(self, message_id: str, to: str, comment: str = "") -> None:
+        """Forward a message."""
+        token = await self._acquire_token()
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.post(
+                f"https://graph.microsoft.com/v1.0/users/{self._mailbox}"
+                f"/messages/{message_id}/forward",
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                json={
+                    "comment": comment,
+                    "toRecipients": [{"emailAddress": {"address": to}}],
+                },
+            )
+            resp.raise_for_status()
+
+    async def move_message(self, message_id: str, folder: str) -> None:
+        """Move a message to a folder."""
+        token = await self._acquire_token()
+
+        async with httpx.AsyncClient() as client:
+            # Get or create folder
+            folder_id = await self._get_or_create_folder(folder, token)
+            resp = await client.post(
+                f"https://graph.microsoft.com/v1.0/users/{self._mailbox}"
+                f"/messages/{message_id}/move",
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                json={"destinationId": folder_id},
+            )
+            resp.raise_for_status()
+
+    async def mark_read(self, message_id: str) -> None:
+        """Mark a message as read."""
+        token = await self._acquire_token()
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.patch(
+                f"https://graph.microsoft.com/v1.0/users/{self._mailbox}"
+                f"/messages/{message_id}",
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                json={"isRead": True},
+            )
+            resp.raise_for_status()
+
+    async def _get_or_create_folder(self, folder_name: str, token: str) -> str:
+        """Get folder ID by name, creating it if needed."""
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(
+                f"https://graph.microsoft.com/v1.0/users/{self._mailbox}/mailFolders"
+                f"?$filter=displayName eq '{folder_name}'",
+                headers={"Authorization": f"Bearer {token}"},
+            )
+            resp.raise_for_status()
+            folders = resp.json().get("value", [])
+            if folders:
+                return folders[0]["id"]
+
+            # Create folder
+            resp = await client.post(
+                f"https://graph.microsoft.com/v1.0/users/{self._mailbox}/mailFolders",
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                json={"displayName": folder_name},
+            )
+            resp.raise_for_status()
+            return resp.json()["id"]