beamflow-runtime 0.3.0__tar.gz

beamflow_runtime-0.3.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.3
+Name: beamflow-runtime
+Version: 0.3.0
+Summary: Actual runtime engine and bootstrap layer for the Beamflow platform.
+Author: juraj.bezdek@gmail.com
+Author-email: juraj.bezdek@gmail.com
+Requires-Python: >=3.11
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: apscheduler (>=3.11.0)
+Requires-Dist: beamflow-clients (>=0.3.0,<0.4.0)
+Requires-Dist: beamflow-lib (>=0.3.0,<0.4.0)
+Requires-Dist: dramatiq (>=1.16.0)
+Requires-Dist: fastapi (>=0.110.0)
+Requires-Dist: setuptools (<70.0.0)
+Requires-Dist: uvicorn (>=0.29.0)

beamflow_runtime-0.3.0/beamflow_runtime/__init__.py

@@ -0,0 +1,39 @@
+"""
+Framework Runtime - Pure glue around beamflow_lib.
+
+This package provides:
+- Ingress API helpers (FastAPI router generation from webhook registry)
+- Worker bootstrap (Dramatiq broker configuration with context middleware)
+- Queue backends (Dramatiq implementation moved from beamflow_lib)
+
+Usage for Ingress API:
+    from beamflow_runtime.ingress.app import create_fastapi_app
+    from beamflow_lib.config.loader import load_runtime_config
+    
+    config = load_runtime_config("/path/to/config")
+    app = create_fastapi_app(config)
+
+Usage for Worker:
+    from beamflow_runtime.worker.entrypoint import configure_worker_process
+    from beamflow_lib.config.loader import load_runtime_config
+    
+    config = load_runtime_config("/path/to/config")
+    runtime = configure_worker_process(config)
+"""
+
+# Main exports
+from .wiring.runtime import Runtime, initialize_runtime, build_worker_runtime
+from .ingress.app import create_fastapi_app
+from .ingress.router import build_webhook_router
+from .worker.runner import run_worker, build_worker_consumer
+
+__all__ = [
+    "Runtime",
+    "initialize_runtime",
+    
+    "build_worker_runtime",
+    "create_fastapi_app",
+    "build_webhook_router",
+    "run_worker",
+    "build_worker_consumer",
+]

beamflow_runtime-0.3.0/beamflow_runtime/ingress/__init__.py

@@ -0,0 +1,2 @@
+# Ingress Module
+# FastAPI router generation from webhook registry

beamflow_runtime-0.3.0/beamflow_runtime/ingress/app.py

@@ -0,0 +1,74 @@
+"""
+FastAPI application factory for webhook ingress.
+
+Provides a convenience function to create a fully configured FastAPI app.
+"""
+from fastapi import FastAPI
+from beamflow_lib.config.runtime_config import RuntimeConfig
+from .router import build_webhook_router
+from ..wiring.runtime import initialize_runtime
+
+
+from typing import List, Union, Optional
+from pathlib import Path
+
+def create_fastapi_app(
+    config: RuntimeConfig, 
+    auto_import: Optional[List[Union[str, Path]]] = None
+) -> FastAPI:
+    """
+    Create FastAPI app with webhook routes.
+    
+    Usage in ingress service:
+        from beamflow_runtime.ingress.app import create_fastapi_app
+        from beamflow_lib.config.loader import load_runtime_config
+        
+        config = load_runtime_config("/path/to/config")
+        app = create_fastapi_app(config)
+    
+    Args:
+        config: RuntimeConfig with webhooks configuration
+        auto_import: List of paths to auto-import modules from
+        
+    Returns:
+        FastAPI application with webhook routes included
+    """
+    # Build runtime (imports modules, sets up backend)
+    rt = initialize_runtime(config, auto_import=auto_import)
+    
+    # Create app
+    app = FastAPI(title="Ingress API")
+    
+    # Include webhook router with configured prefix
+    app.include_router(build_webhook_router(), prefix=config.webhooks.prefix)
+    
+    # Store runtime reference on app for access in middleware/dependencies if needed
+    app.state.runtime = rt
+    
+    import logging
+    import sys
+    app_logger = logging.getLogger(__name__)
+    app_logger.setLevel(logging.INFO)
+    if not app_logger.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(logging.Formatter('INFO:     %(message)s'))
+        app_logger.addHandler(handler)
+
+    # Include auto-imported routers
+    if hasattr(rt, 'imported_modules'):
+        for mod in rt.imported_modules:
+            if hasattr(mod, "router"):
+                app.include_router(mod.router)
+                mod_file = getattr(mod, "__file__", mod.__name__)
+                app_logger.info(f"router automimpoted from [{mod_file}]")
+                
+    app_logger.info("Registered paths:")
+    for route in app.routes:
+        methods = getattr(route, "methods", None)
+        path = getattr(route, "path", None)
+        name = getattr(route, "name", getattr(route, "endpoint", "").__name__ if hasattr(route, "endpoint") else "")
+        if path:
+            methods_str = ",".join(methods) if methods else "ANY"
+            app_logger.info(f"- {methods_str} {path} [{name}]")
+    
+    return app

beamflow_runtime-0.3.0/beamflow_runtime/ingress/router.py

@@ -0,0 +1,68 @@
+"""
+Webhook router generation from beamflow_lib registry.
+
+Builds FastAPI routes from webhooks registered with @ingress.webhook decorator.
+"""
+import inspect
+from typing import Any, Optional
+from fastapi import APIRouter, Request, Response
+from beamflow_lib.pipelines.ingress import ingress
+from beamflow_lib.context import integration_context
+
+
+async def _invoke_handler(handler, request: Request, body: bytes) -> Any:
+    """
+    Invoke a webhook handler within the integration context.
+    
+    Handles both sync and async handlers.
+    """
+    metadata = handler._ingress_metadata
+    
+    with integration_context(
+        integration=metadata["integration"],
+        integration_pipeline=metadata["pipeline"]
+    ):
+        # Pass the request to the handler - let the handler decide what to do with it
+        if inspect.iscoroutinefunction(handler):
+            result = await handler(request)
+        else:
+            result = handler(request)
+        
+        return result
+
+
+def build_webhook_router() -> APIRouter:
+    """
+    Build FastAPI router from registered webhooks.
+    
+    Reads all webhooks registered via @ingress.webhook and creates
+    FastAPI endpoints for each one.
+    
+    Returns:
+        APIRouter with webhook routes tagged under "webhooks"
+    """
+    router = APIRouter(tags=["webhooks"])
+    
+    for handler in ingress.get_webhooks():
+        metadata = handler._ingress_metadata
+        # Build path from metadata
+        path = metadata["path"]
+        methods = [metadata.get("method", "POST")]
+
+        # Create endpoint closure that captures the handler
+        # Define the endpoint function factory
+        def create_endpoint(h=handler):
+            async def endpoint(request: Request) -> Response:
+                result = await _invoke_handler(h, request, await request.body())
+                # Default to 202 Accepted for webhook processing
+                if result is None:
+                    return Response(status_code=202)
+                return result
+            return endpoint
+
+        # Create the endpoint function
+        endpoint = create_endpoint(handler)
+        
+        router.add_api_route(path, endpoint, methods=methods)
+    
+    return router

beamflow_runtime-0.3.0/beamflow_runtime/wiring/__init__.py

@@ -0,0 +1,2 @@
+# Wiring Module
+# Runtime configuration and bootstrap utilities

beamflow_runtime-0.3.0/beamflow_runtime/wiring/runtime.py

@@ -0,0 +1,161 @@
+import os
+from pathlib import Path
+from dataclasses import dataclass, field
+from typing import Any, Optional, List, Union
+import importlib
+import importlib.util
+import dramatiq
+from dramatiq.brokers.redis import RedisBroker
+from beamflow_lib.config.runtime_config import RuntimeConfig, BackendType
+from beamflow_lib.queue.backend import set_backend, TaskBackend
+from beamflow_runtime.worker.backends.dramatiq.dramatiq_backend import DramatiqBackend, FrameworkContextMiddleware
+
+
+@dataclass
+class Runtime:
+    """Runtime container for backend and config references."""
+    backend: TaskBackend
+    config: RuntimeConfig
+    imported_modules: List[Any] = field(default_factory=list)
+
+    @property
+    def broker(self):
+        """Return the global Dramatiq broker."""
+        import dramatiq
+        return dramatiq.get_broker()
+
+
+def _auto_import_path(path: Union[str, Path]) -> List[Any]:
+    """
+    Import modules from a file or directory path.
+    
+    If path is a directory:
+    - If it contains __init__.py, import it as a module.
+    - Otherwise, import all *.py files in it.
+    If path is a .py file, import it.
+    """
+    p = Path(path)
+    if not p.exists():
+        return []
+
+    modules = []
+    if p.is_dir():
+        if (p / "__init__.py").exists():
+            mod = _import_module_from_path(p)
+            if mod:
+                modules.append(mod)
+        else:
+            for f in sorted(p.glob("*.py")):
+                if f.name != "__init__.py":
+                    mod = _import_module_from_path(f)
+                    if mod:
+                        modules.append(mod)
+    elif p.is_file() and p.suffix == ".py":
+        mod = _import_module_from_path(p)
+        if mod:
+            modules.append(mod)
+            
+    return modules
+
+
+def _import_module_from_path(file_path: Path) -> Any:
+    """Import a module from a file path dynamically."""
+    import sys
+    # Use parent directory name if it's an __init__.py file for package names
+    if file_path.name == "__init__.py":
+        module_name = file_path.parent.name
+    elif file_path.is_dir():
+        module_name = file_path.name
+        file_path = file_path / "__init__.py"
+    else:
+        module_name = file_path.stem
+        
+    spec = importlib.util.spec_from_file_location(module_name, str(file_path))
+    if spec and spec.loader:
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        return module
+    
+    return None
+
+
+def _build_redis_url(config: RuntimeConfig) -> str:
+    """Build Redis URL from config or environment."""
+    if os.getenv("REDIS_URL"):
+        return os.getenv("REDIS_URL")
+        
+    if config.backend.dramatiq and config.backend.dramatiq.redis_url:
+        return config.backend.dramatiq.redis_url
+        
+    return "redis://localhost:6379/0"
+
+
+def initialize_runtime(
+    config: RuntimeConfig, 
+    auto_import: Optional[List[Union[str, Path]]] = None
+) -> Runtime:
+    """
+    Initialize the framework runtime: configuration, clients, and task backend.
+    
+    This is the single entry point for setting up the producer side (backend submission)
+    and registry initialization.
+    """
+    # 1. Init Client Registry
+    from beamflow_clients.registry import init_registry
+    init_registry(config)
+
+    # 2. Handle auto-imports from paths
+    imported_modules = []
+    if auto_import:
+        for path in auto_import:
+            mods = _auto_import_path(path)
+            if mods:
+                imported_modules.extend(mods)
+
+    # 3. Configure task backend based on config.backend.type
+    backend: Optional[TaskBackend] = None
+    if config.backend.type == BackendType.DRAMATIQ:
+        redis_url = _build_redis_url(config)
+        broker = RedisBroker(url=redis_url)
+        dramatiq.set_broker(broker)
+        
+        # Dramatiq's built-in middlewares (like Prometheus) expect the process_boot 
+        # signal to be emitted to initialize their internal states (like prometheus Counters).
+        # We must emit this manually here because we are initializing the broker programmatically 
+        # (not via dramatiq CLI), ensuring producers (API) also initialize metrics.
+        broker.emit_after("process_boot")
+        
+        # Set beamflow_lib.queue.backend
+        backend = set_backend(DramatiqBackend())
+    elif config.backend.type == BackendType.ASYNC:
+        from beamflow_lib.queue.asyncio_backend import AsyncioBackend
+        backend = set_backend(AsyncioBackend())
+    elif config.backend.type == BackendType.MANAGED:
+        # Managed backend might require environment variables for its URL
+        from beamflow_lib.queue.backends.managed_tasks import ManagedTasksBackend
+        backend = set_backend(ManagedTasksBackend(
+            api_url=os.getenv("MANAGED_API_URL", ""),
+            auth_token=os.getenv("MANAGED_AUTH_TOKEN", ""),
+            service_url=os.getenv("MANAGED_SERVICE_URL"),
+        ))
+    else:
+        raise ValueError(f"Unsupported backend: {config.backend.type}")
+    
+    return Runtime(backend=backend, config=config, imported_modules=imported_modules)
+
+
+
+def build_worker_runtime(
+    config: RuntimeConfig, 
+    auto_import: Optional[List[Union[str, Path]]] = None
+) -> Runtime:
+    """
+    Configure runtime for worker service.
+    
+    Deprecated: Use initialize_runtime(config) and then get a consumer.
+    """
+    # Import task modules is now out of scope for RuntimeConfig, 
+    # and they should be passed as auto_import paths or imported manually.
+
+    return initialize_runtime(config, auto_import=auto_import)

beamflow_runtime-0.3.0/beamflow_runtime/worker/__init__.py

@@ -0,0 +1,2 @@
+# Worker Module
+# Provides bootstrap and entrypoint utilities for Dramatiq workers

beamflow_runtime-0.3.0/beamflow_runtime/worker/backends/dramatiq/bootstrap.py

@@ -0,0 +1,57 @@
+"""
+Worker bootstrap utilities.
+
+Provides functions to configure the Dramatiq broker with framework middleware.
+"""
+import dramatiq
+from dramatiq.brokers.redis import RedisBroker
+from beamflow_lib.config.runtime_config import RuntimeConfig, BackendType
+from beamflow_lib.queue.backend import set_backend
+from .dramatiq_backend import DramatiqBackend, FrameworkContextMiddleware
+
+
+def _build_redis_url(config: RuntimeConfig) -> str:
+    """Build Redis URL from config or environment."""
+    if os.getenv("REDIS_URL"):
+        return os.getenv("REDIS_URL")
+        
+    if config.backend.dramatiq and config.backend.dramatiq.redis_url:
+        return config.backend.dramatiq.redis_url
+        
+    return "redis://localhost:6379/0"
+
+
+def configure_worker(config: RuntimeConfig):
+    """
+    Configure Dramatiq broker with context middleware.
+    
+    This sets up the broker and middleware for worker processes.
+    Note: This is a lower-level function. For most cases, use
+    `configure_worker_process()` from entrypoint.py instead.
+    
+    Args:
+        config: RuntimeConfig with worker and redis configuration
+        
+    Returns:
+        The configured broker instance
+        
+    Raises:
+        ValueError: If backend type is not DRAMATIQ
+    """
+    if config.backend.type != BackendType.DRAMATIQ:
+        raise ValueError(f"Unsupported backend for worker: {config.backend.type}")
+    
+    redis_url = _build_redis_url(config)
+    broker = RedisBroker(url=redis_url)
+    broker.add_middleware(FrameworkContextMiddleware())
+    dramatiq.set_broker(broker)
+    set_backend(DramatiqBackend())
+    
+    # Dramatiq's built-in middlewares (like Prometheus) expect the process_boot 
+    # signal to be emitted to initialize their internal states (like prometheus Counters).
+    # Since we initialize the Worker programmatically (not via dramatiq CLI),
+    # we must emit this manually. We emit it here so that producers (API) also 
+    # initialize the metrics, preventing crashes during enqueue.
+    broker.emit_after("process_boot")
+    
+    return broker

beamflow_runtime-0.3.0/beamflow_runtime/worker/backends/dramatiq/consumer_middleware.py

@@ -0,0 +1,48 @@
+import threading
+import asyncio
+import logging
+import dramatiq
+from beamflow_lib.pipelines.consumer import ConsumerRunner
+
+logger = logging.getLogger(__name__)
+
+class ConsumerMiddleware(dramatiq.Middleware):
+    def __init__(self):
+        self._loop: asyncio.AbstractEventLoop = None
+        self._thread: threading.Thread = None
+        self._runner: ConsumerRunner = None
+        self._stop_event = threading.Event()
+
+    def after_worker_boot(self, broker, worker):
+        """Called when the worker process starts."""
+        logger.info("Starting ConsumerRunner thread...")
+        
+        self._thread = threading.Thread(target=self._run_loop, daemon=True, name="ConsumerRunnerThread")
+        self._thread.start()
+
+    def before_worker_shutdown(self, broker, worker):
+        """Called when the worker process is shutting down."""
+        logger.info("Stopping ConsumerRunner thread...")
+        if self._loop:
+            asyncio.run_coroutine_threadsafe(self._runner.stop(), self._loop)
+            # Give it a moment to stop
+            self._thread.join(timeout=5.0)
+
+    def _run_loop(self):
+        """Runs the asyncio loop in a separate thread."""
+        logger.info("ConsumerRunner loop started.")
+        self._loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self._loop)
+        
+        self._runner = ConsumerRunner()
+        
+        self._loop.run_until_complete(self._runner.start())
+        
+        # Keep loop running until stopped
+        try:
+            self._loop.run_forever()
+        except Exception as e:
+            logger.error(f"ConsumerRunner loop error: {e}")
+        finally:
+            self._loop.close()
+            logger.info("ConsumerRunner loop closed.")