dory-processor-sdk 0.0.1__py3-none-any.whl

dory/__init__.py

@@ -0,0 +1,101 @@
+"""
+Dory SDK - Python Integration Package for Processor Applications
+
+This SDK handles all operational concerns (graceful shutdown, state migration,
+health checks, observability) so developers focus solely on business logic.
+
+Quick Start (Class-based API):
+    from dory import DoryApp, BaseProcessor, stateful
+
+    class MyProcessor(BaseProcessor):
+        counter = stateful(0)  # Auto-saved/restored
+
+        async def run(self):
+            async for _ in self.run_loop(interval=1):
+                self.counter += 1
+
+    if __name__ == '__main__':
+        DoryApp().run(MyProcessor)
+
+Quick Start (Function-based API):
+    from dory.simple import processor, state
+
+    counter = state(0)
+
+    @processor
+    async def main(ctx):
+        async for _ in ctx.run_loop(interval=1):
+            counter.value += 1
+"""
+
+__version__ = "0.0.1"
+
+# Core API
+from dory.core.processor import BaseProcessor
+from dory.core.context import ExecutionContext
+from dory.core.app import DoryApp
+
+# Configuration
+from dory.config.schema import DoryConfig
+
+# Decorators for simplified integration
+from dory.decorators import stateful, StatefulVar
+
+# Exceptions
+from dory.utils.errors import (
+    DoryError,
+    DoryStartupError,
+    DoryShutdownError,
+    DoryStateError,
+    DoryConfigError,
+)
+
+# Edge node support (fencing, split-brain prevention, heartbeat)
+from dory.edge import (
+    FencingManager,
+    FencingConfig,
+    FencingToken,
+    FencingError,
+    FenceViolation,
+    StaleEpochError,
+    ProcessorRole,
+    RoleManager,
+    RoleTransition,
+    HeartbeatManager,
+    HeartbeatConfig,
+    ConnectivityStatus,
+    EdgeHealthReporter,
+)
+
+__all__ = [
+    # Core API
+    "BaseProcessor",
+    "ExecutionContext",
+    "DoryApp",
+    "DoryConfig",
+    # Decorators
+    "stateful",
+    "StatefulVar",
+    # Exceptions
+    "DoryError",
+    "DoryStartupError",
+    "DoryShutdownError",
+    "DoryStateError",
+    "DoryConfigError",
+    # Edge node support
+    "FencingManager",
+    "FencingConfig",
+    "FencingToken",
+    "FencingError",
+    "FenceViolation",
+    "StaleEpochError",
+    "ProcessorRole",
+    "RoleManager",
+    "RoleTransition",
+    "HeartbeatManager",
+    "HeartbeatConfig",
+    "ConnectivityStatus",
+    "EdgeHealthReporter",
+    # Version
+    "__version__",
+]

dory/auth/__init__.py

@@ -0,0 +1,10 @@
+"""Authentication providers for Dory SDK."""
+
+from dory.auth.oauth2 import OAuth2Error, OAuth2TokenProvider, TokenProvider
+
+
+__all__ = [
+    "OAuth2Error",
+    "OAuth2TokenProvider",
+    "TokenProvider",
+]

dory/auth/oauth2.py

@@ -0,0 +1,153 @@
+"""OAuth2 client credentials token provider with caching and auto-refresh."""
+
+import logging
+import time
+from typing import Protocol, runtime_checkable
+from urllib.parse import quote
+
+import aiohttp
+
+from dory.utils.errors import DoryConfigError
+
+
+logger = logging.getLogger(__name__)
+
+
+class OAuth2Error(Exception):
+    """Raised when an OAuth2 token request fails."""
+
+    def __init__(self, message: str, status_code: int | None = None):
+        super().__init__(message)
+        self.message = message
+        self.status_code = status_code
+
+
+@runtime_checkable
+class TokenProvider(Protocol):
+    """Protocol for token providers.
+
+    Implement this protocol to provide custom token sources
+    (e.g., Vault, AWS IAM) without being locked to OAuth2.
+    """
+
+    async def get_token(self) -> str: ...
+
+
+class OAuth2TokenProvider:
+    """Async OAuth2 client credentials token provider with caching and auto-refresh.
+
+    Fetches tokens from an OAuth2 token endpoint using the client credentials
+    grant type. Tokens are cached in memory and automatically refreshed before
+    expiry based on the configured buffer.
+
+    Args:
+        token_url: OAuth2 token endpoint URL.
+        client_id: OAuth2 client ID.
+        client_secret: OAuth2 client secret.
+        scopes: List of OAuth2 scopes to request.
+        refresh_buffer_sec: Seconds before expiry to trigger a refresh.
+    """
+
+    def __init__(
+        self,
+        token_url: str,
+        client_id: str,
+        client_secret: str,
+        scopes: list[str] | None = None,
+        refresh_buffer_sec: int = 60,
+    ):
+        if not token_url:
+            raise DoryConfigError("OAuth2 token_url is required")
+        if not client_id:
+            raise DoryConfigError("OAuth2 client_id is required")
+        if not client_secret:
+            raise DoryConfigError("OAuth2 client_secret is required")
+
+        self._token_url = token_url
+        self._client_id = client_id
+        self._client_secret = client_secret
+        self._scopes = scopes or []
+        self._refresh_buffer_sec = refresh_buffer_sec
+
+        self._cached_token: str | None = None
+        self._token_expiry: float = 0.0
+
+    async def get_token(self) -> str:
+        """Return a valid access token, fetching or refreshing as needed.
+
+        Returns:
+            A valid OAuth2 access token string.
+
+        Raises:
+            OAuth2Error: If the token request fails.
+        """
+        now = time.monotonic()
+        if self._cached_token and now < (self._token_expiry - self._refresh_buffer_sec):
+            return self._cached_token
+
+        return await self._fetch_token()
+
+    async def _fetch_token(self) -> str:
+        """Fetch a new token from the OAuth2 token endpoint."""
+        data: dict[str, str] = {
+            "grant_type": "client_credentials",
+        }
+        if self._scopes:
+            data["scope"] = " ".join(self._scopes)
+
+        # Use HTTP Basic Auth (RFC 6749 §2.3.1) as required by Cognito
+        auth = aiohttp.BasicAuth(self._client_id, self._client_secret)
+
+        try:
+            async with aiohttp.ClientSession() as session, session.post(self._token_url, data=data, auth=auth) as resp:
+                if resp.status != 200:
+                    body = await resp.text()
+                    raise OAuth2Error(
+                        f"Token request failed (HTTP {resp.status}): {body}",
+                        status_code=resp.status,
+                    )
+
+                result = await resp.json()
+        except OAuth2Error:
+            raise
+        except Exception as e:
+            raise OAuth2Error(f"Token request failed: {e}") from e
+
+        access_token = result.get("access_token")
+        if not access_token:
+            raise OAuth2Error("Token response missing 'access_token' field")
+
+        expires_in = result.get("expires_in", 3600)
+        self._cached_token = str(access_token)
+        self._token_expiry = time.monotonic() + expires_in
+
+        logger.info(f"OAuth2 token acquired, expires in {expires_in}s")
+        return self._cached_token
+
+    async def build_amqp_url(
+        self,
+        host: str,
+        port: int = 5671,
+        vhost: str = "/",
+        tls: bool = True,
+    ) -> str:
+        """Build an AMQP URL using the current token as password.
+
+        Args:
+            host: RabbitMQ hostname.
+            port: RabbitMQ port (default 5671 for TLS).
+            vhost: RabbitMQ virtual host.
+            tls: Whether to use amqps:// (TLS) or amqp://.
+
+        Returns:
+            A fully-formed AMQP connection URL.
+        """
+        token = await self.get_token()
+        scheme = "amqps" if tls else "amqp"
+        encoded_token = quote(token, safe="")
+        # Use empty username with token as password (RabbitMQ OAuth2 convention)
+        if vhost == "/":
+            vhost_path = "%2f"
+        else:
+            vhost_path = quote(vhost, safe="")
+        return f"{scheme}://:{encoded_token}@{host}:{port}/{vhost_path}"

dory/auto_instrument.py

@@ -0,0 +1,142 @@
+"""
+Auto-instrumentation decorator for handlers.
+
+Automatically applies:
+- Request ID generation
+- Request tracking
+- OpenTelemetry span creation
+- Error classification
+- Attribute injection
+
+No manual decorators needed!
+"""
+
+import functools
+import logging
+from typing import Callable, Any
+
+logger = logging.getLogger(__name__)
+
+
+def auto_instrument(func: Callable) -> Callable:
+    """
+    Auto-instrument async function with all SDK features.
+
+    Automatically handles:
+    - Request ID generation
+    - Request tracking with timeout
+    - OpenTelemetry span creation
+    - Span attributes injection
+    - Error classification and logging
+
+    Usage:
+        @auto_instrument
+        async def handler(self, request):
+            # All instrumentation is automatic!
+            return {"status": "ok"}
+
+    Or with metaclass (no decorator needed):
+        class MyProcessor(BaseProcessor):
+            # All handle_* methods automatically instrumented!
+            async def handle_request(self, request):
+                return {"status": "ok"}
+
+    Args:
+        func: Async function to instrument
+
+    Returns:
+        Instrumented async function
+    """
+
+    @functools.wraps(func)
+    async def wrapper(self, *args, **kwargs):
+        # Extract request if present (first arg or keyword arg)
+        request = None
+        if args:
+            request = args[0]
+        elif "request" in kwargs:
+            request = kwargs["request"]
+
+        # Get processor components (auto-initialized by BaseProcessor)
+        request_id_middleware = getattr(self, "request_id_middleware", None)
+        request_tracker = getattr(self, "request_tracker", None)
+        otel = getattr(self, "otel", None)
+        error_classifier = getattr(self, "error_classifier", None)
+
+        # 1. Generate request ID
+        request_id = None
+        if request_id_middleware:
+            request_id = request_id_middleware.generate_id()
+            # Store in request for retrieval
+            if request is not None and hasattr(request, "__setitem__"):
+                request["request_id"] = request_id
+            elif request is not None and hasattr(request, "__dict__"):
+                request.request_id = request_id
+            logger.debug(f"Generated request ID: {request_id}")
+
+        # 2. Track request
+        request_tracker_ctx = None
+        if request_tracker:
+            request_tracker_ctx = request_tracker.track_request(request_id)
+            await request_tracker_ctx.__aenter__()
+            logger.debug(f"Started tracking request: {request_id}")
+
+        # 3. Create OpenTelemetry span
+        span_ctx = None
+        if otel:
+            span_name = f"{self.__class__.__name__}.{func.__name__}"
+            attributes = {
+                "function": func.__name__,
+                "class": self.__class__.__name__,
+            }
+            if request_id:
+                attributes["request.id"] = request_id
+            if request is not None and hasattr(request, "path"):
+                attributes["endpoint"] = str(request.path)
+            if request is not None and hasattr(request, "method"):
+                attributes["http.method"] = str(request.method)
+
+            span_ctx = otel.create_span(span_name, attributes=attributes)
+            span_ctx.__enter__()
+            logger.debug(f"Created span: {span_name}")
+
+        try:
+            # Execute handler
+            result = await func(self, *args, **kwargs)
+
+            # Mark request as successful
+            if request_tracker_ctx:
+                await request_tracker_ctx.__aexit__(None, None, None)
+                logger.debug(f"Request completed successfully: {request_id}")
+
+            if span_ctx:
+                span_ctx.__exit__(None, None, None)
+
+            return result
+
+        except Exception as e:
+            # Classify error
+            if error_classifier:
+                error_info = error_classifier.classify(e)
+                logger.warning(
+                    f"Handler error: {error_info.error_type.value} - {e}",
+                    extra={
+                        "request_id": request_id,
+                        "error_type": error_info.error_type.value,
+                        "is_transient": error_info.is_transient,
+                    },
+                )
+            else:
+                logger.warning(f"Handler error: {e}", extra={"request_id": request_id})
+
+            # Mark request as failed
+            if request_tracker_ctx:
+                await request_tracker_ctx.__aexit__(type(e), e, None)
+
+            # Record exception in span
+            if span_ctx:
+                span_ctx.__exit__(type(e), e, None)
+
+            raise
+
+    return wrapper

dory/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Dory SDK CLI tools."""
+
+from dory.cli.main import main
+
+__all__ = ["main"]

dory/cli/main.py

@@ -0,0 +1,137 @@
+"""
+Dory SDK CLI - Command line tools for Dory SDK.
+
+Provides commands for:
+- Initializing new Dory projects
+- Validating configuration
+
+Usage:
+    dory init my-app
+    dory validate
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from dory import __version__
+from dory.cli.templates import (
+    generate_dockerfile,
+    generate_processor_template,
+)
+
+
+def cmd_init(args: argparse.Namespace) -> int:
+    """Initialize a new Dory project."""
+    name = args.name
+    output_dir = Path(args.output or ".")
+
+    print(f"Initializing Dory project: {name}")
+
+    # Generate files
+    files = {
+        "main.py": generate_processor_template(name),
+        "Dockerfile": generate_dockerfile(name),
+    }
+
+    for filename, content in files.items():
+        filepath = output_dir / filename
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+
+        if filepath.exists() and not args.force:
+            print(f"  Skipping {filename} (already exists, use --force to overwrite)")
+            continue
+
+        filepath.write_text(content)
+        print(f"  Created {filename}")
+
+    print()
+    print("Next steps:")
+    print(f"  1. Edit main.py with your processor logic")
+    print(f"  2. Build: docker build -t {name}:latest .")
+    print(f"  3. Register the processor in the orchestrator database")
+    print()
+
+    return 0
+
+
+def cmd_validate(args: argparse.Namespace) -> int:
+    """Validate Dory configuration."""
+    from dory.config.loader import ConfigLoader
+
+    config_file = args.config
+
+    try:
+        loader = ConfigLoader(config_file=config_file)
+        config = loader.load()
+        print("Configuration is valid!")
+        print()
+        print("Current settings:")
+        for key, value in config.model_dump().items():
+            print(f"  {key}: {value}")
+        return 0
+    except Exception as e:
+        print(f"Configuration error: {e}")
+        return 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        prog="dory",
+        description="Dory SDK CLI - Tools for building stateful Kubernetes processors",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Commands")
+
+    # init command
+    init_parser = subparsers.add_parser(
+        "init",
+        help="Initialize a new Dory project",
+    )
+    init_parser.add_argument(
+        "name",
+        help="Project/app name",
+    )
+    init_parser.add_argument(
+        "-o", "--output",
+        help="Output directory (default: current directory)",
+    )
+    init_parser.add_argument(
+        "-i", "--image",
+        help="Docker image name (default: <name>:latest)",
+    )
+    init_parser.add_argument(
+        "-f", "--force",
+        action="store_true",
+        help="Overwrite existing files",
+    )
+    init_parser.set_defaults(func=cmd_init)
+
+    # validate command
+    validate_parser = subparsers.add_parser(
+        "validate",
+        help="Validate Dory configuration",
+    )
+    validate_parser.add_argument(
+        "-c", "--config",
+        help="Path to config file (default: dory.yaml)",
+    )
+    validate_parser.set_defaults(func=cmd_validate)
+
+    args = parser.parse_args(argv)
+
+    if not args.command:
+        parser.print_help()
+        return 0
+
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dory/cli/templates.py

@@ -0,0 +1,123 @@
+"""
+Templates for generating project files.
+"""
+
+
+def generate_dockerfile(name: str) -> str:
+    """Generate Dockerfile for a Dory processor."""
+    return f'''# Dockerfile for Dory processor: {name}
+
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \\
+    curl \\
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Dory SDK for EKS deployment
+RUN pip install --no-cache-dir dory-processor-sdk[production]
+
+# Copy application
+COPY main.py .
+
+# Expose ports (8080 for health, 8081 for app)
+EXPOSE 8080 8081
+
+# Health check
+HEALTHCHECK --interval=10s --timeout=5s --start-period=5s --retries=3 \\
+    CMD curl -f http://localhost:8080/health || exit 1
+
+# Run the application
+CMD ["python", "main.py"]
+'''
+
+
+def generate_processor_template(name: str) -> str:
+    """Generate a template processor Python file."""
+    class_name = "".join(word.capitalize() for word in name.replace("-", "_").split("_"))
+
+    return f'''"""
+{name} - A Dory-powered stateful processor.
+
+Features:
+- Zero-downtime migration
+- Automatic state persistence
+- Graceful shutdown
+- Health monitoring
+
+Generated by: dory init {name}
+"""
+
+import asyncio
+from dory import DoryApp, BaseProcessor, stateful
+
+
+class {class_name}(BaseProcessor):
+    """
+    Your processor implementation.
+
+    The @stateful decorator automatically handles state save/restore.
+    Just implement the run() method with your business logic.
+    """
+
+    # Stateful variables (automatically saved and restored)
+    counter = stateful(0)
+    data = stateful(dict)
+
+    async def startup(self) -> None:
+        """Initialize resources (optional)."""
+        self.context.logger().info("Starting up...")
+        # Load models, open connections, etc.
+
+    async def run(self) -> None:
+        """Main processing loop."""
+        logger = self.context.logger()
+
+        # Use run_loop() for automatic shutdown detection
+        async for i in self.run_loop(interval=1):
+            self.counter += 1
+            logger.info(f"Iteration {{i}}: counter={{self.counter}}")
+
+            # Your business logic here
+            # ...
+
+    async def shutdown(self) -> None:
+        """Cleanup resources (optional)."""
+        self.context.logger().info(f"Shutting down. Final counter: {{self.counter}}")
+        # Close connections, flush buffers, etc.
+
+
+if __name__ == "__main__":
+    DoryApp().run({class_name})
+'''
+
+
+def generate_simple_processor_template(name: str) -> str:
+    """Generate a minimal processor using function-based API."""
+    return f'''"""
+{name} - A minimal Dory processor using function-based API.
+
+Generated by: dory init {name} --simple
+"""
+
+from dory.simple import processor, state
+
+# State variables (automatically saved and restored)
+counter = state(0)
+data = state(dict)
+
+
+@processor
+async def main(ctx):
+    """Main processing loop."""
+    logger = ctx.logger()
+
+    async for i in ctx.run_loop(interval=1):
+        counter.value += 1
+        logger.info(f"Iteration {{i}}: counter={{counter.value}}")
+
+        # Your business logic here
+        # ...
+'''

dory/config/__init__.py

@@ -0,0 +1,23 @@
+"""Configuration loading and schema definitions."""
+
+from dory.config.schema import DoryConfig
+from dory.config.loader import ConfigLoader
+from dory.config.defaults import DEFAULT_CONFIG
+from dory.config.presets import (
+    get_preset,
+    list_presets,
+    DEVELOPMENT_PRESET,
+    PRODUCTION_PRESET,
+    HIGH_AVAILABILITY_PRESET,
+)
+
+__all__ = [
+    "DoryConfig",
+    "ConfigLoader",
+    "DEFAULT_CONFIG",
+    "get_preset",
+    "list_presets",
+    "DEVELOPMENT_PRESET",
+    "PRODUCTION_PRESET",
+    "HIGH_AVAILABILITY_PRESET",
+]