fastprocesses 0.7.4__py3-none-any.whl

fastprocesses/__init__.py

@@ -0,0 +1,8 @@
+"""fastProcesses package.
+
+A library to create a FastAPI-based  OGC API Processes wrapper around existing projects.
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

fastprocesses/api/manager.py

@@ -0,0 +1,377 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+import json
+from typing import Any, Dict, List, Tuple
+
+from celery.result import AsyncResult
+
+from fastprocesses.common import redis_cache
+from fastprocesses.core.logging import logger
+from fastprocesses.core.models import (
+    CalculationTask,
+    ExecutionMode,
+    ProcessDescription,
+    ProcessExecRequestBody,
+    ProcessExecResponse,
+)
+from fastprocesses.processes.process_registry import get_process_registry
+from fastprocesses.common import celery_app
+
+
+class ExecutionStrategy(ABC):
+    """
+    Abstract base class implementing the Strategy pattern for process execution.
+    Different execution modes (sync/async) implement this interface.
+    """
+    
+    def __init__(self, process_manager):
+        self.process_manager: ProcessManager = process_manager
+
+    @abstractmethod
+    def execute(self, process_id: str, calculation_task: CalculationTask) -> ProcessExecResponse:
+        pass
+
+class AsyncExecutionStrategy(ExecutionStrategy):
+    """
+    Handles asynchronous process execution by:
+    1. Submitting task to Celery queue
+    2. Creating initial job status in cache
+    3. Returning immediately with job ID
+    """
+    
+    def execute(self, process_id: str, calculation_task: CalculationTask) -> ProcessExecResponse:
+
+        # dump data to json
+        serialized_data = json.dumps(calculation_task.model_dump(
+                include={"inputs", "outputs", "response"}
+        ))
+        
+        # Submit task to Celery worker queue for background processing
+        task = self.process_manager.celery_app.send_task(
+            'execute_process',
+            args=[process_id, serialized_data]
+        )
+
+        # Initialize job metadata in cache with status 'accepted'
+        job_info = {
+            "status": "accepted",
+            "type": "process",
+            "process_id": process_id,
+            "created": datetime.utcnow().isoformat(),
+            "updated": datetime.utcnow().isoformat(),
+            "progress": 0
+        }
+        self.process_manager.cache.put(f"job:{task.id}", job_info)
+        
+        return ProcessExecResponse(status="accepted", jobID=task.id, type="process")
+
+class SyncExecutionStrategy(ExecutionStrategy):
+    """Strategy for synchronous execution."""
+    
+    def execute(self, process_id: str, calculation_task: CalculationTask) -> ProcessExecResponse:
+        service = self.process_manager.service_registry.get_service(process_id)
+        # TODO: response type and outputs must be passed, too
+        result = service.execute(calculation_task.inputs)
+        
+        task = self.process_manager.celery_app.send_task(
+            'store_result',
+            args=[process_id, calculation_task.celery_key, result]
+        )
+        
+        job_info = {
+            "status": "successful",
+            "type": "process",
+            "process_id": process_id,
+            "created": datetime.utcnow().isoformat(),
+            "started": datetime.utcnow().isoformat(),
+            "finished": datetime.utcnow().isoformat(),
+            "updated": datetime.utcnow().isoformat(),
+            "progress": 100,
+            "result": result
+        }
+        self.process_manager.cache.put(f"job:{task.id}", job_info)
+        
+        return ProcessExecResponse(status="successful", jobID=task.id, type="process", value=result)
+
+class ProcessManager:
+    """Manages processes, including execution, status checking, and job management."""
+
+    def __init__(self):
+        """Initializes the ProcessManager with Celery app and service registry."""
+        self.celery_app = celery_app
+        self.service_registry = get_process_registry()
+        self.cache = redis_cache
+
+    def get_available_processes(
+            self, limit: int, offset: int
+    ) -> Tuple[List[ProcessDescription], str]:
+        logger.info("Retrieving available processes")
+        """
+        Retrieves a list of available processes.
+
+        Returns:
+            List[ProcessDescription]: A list of process descriptions.
+        """
+        process_ids = self.service_registry.get_service_ids()
+        processes = [
+            self.get_process_description(process_id)
+            for process_id
+            in process_ids[offset:offset+limit]
+        ]
+        next_link = None
+        if offset + limit < len(process_ids):
+            next_link = f"/processes?limit={limit}&offset={offset+limit}"
+        return processes, next_link
+
+
+    def get_process_description(self, process_id: str) -> ProcessDescription:
+        logger.info(f"Retrieving description for process ID: {process_id}")
+        """
+        Retrieves the description of a specific process.
+
+        Args:
+            process_id (str): The ID of the process.
+
+        Returns:
+            ProcessDescription: The description of the process.
+
+        Raises:
+            ValueError: If the process is not found.
+        """
+        if not self.service_registry.has_service(process_id):
+            logger.error(f"Process {process_id} not found!")
+            raise ValueError(f"Process {process_id} not found!")
+
+        service = self.service_registry.get_service(process_id)
+        return service.get_description()
+
+    def execute_process(self, process_id: str, data: ProcessExecRequestBody) -> ProcessExecResponse:
+        """
+        Main process execution orchestration:
+        1. Validates process existence and input data
+        2. Checks result cache to avoid recomputation
+        3. Selects execution strategy (sync/async)
+        4. Delegates execution to appropriate strategy
+        
+        Args:
+            process_id: Identifier for the process to execute
+            data: Contains input parameters and execution mode
+            
+        Returns:
+            ProcessExecResponse with job status and ID
+            
+        Raises:
+            ValueError: If process not found or input validation fails
+        """
+        logger.info(f"Executing process ID: {process_id}")
+        
+        # Validate process exists
+        if not self.service_registry.has_service(process_id):
+            logger.error(f"Process {process_id} not found!")
+            raise ValueError(f"Process {process_id} not found!")
+
+        # Get service and validate inputs
+        service = self.service_registry.get_service(process_id)
+        try:
+            service.validate_inputs(data.inputs)
+        except ValueError as e:
+            logger.error(f"Input validation failed for process {process_id}: {str(e)}")
+            raise ValueError(f"Input validation failed: {str(e)}")
+
+        try:
+            service.validate_outputs(data.outputs)
+        except ValueError as e:
+            logger.error(f"Output validation failed for process {process_id}: {str(e)}")
+            raise ValueError(f"Output validation failed: {str(e)}")
+
+        # Create calculation task
+        calculation_task = CalculationTask(
+            inputs=data.inputs,
+            outputs=data.outputs,
+            response=data.response
+        )
+        
+        # Check cache first
+        cached_result = self._check_cache(calculation_task)
+        if cached_result:
+            return cached_result
+        
+        # Select execution strategy based on mode
+        execution_strategies = {
+            ExecutionMode.SYNC: SyncExecutionStrategy(self),
+            ExecutionMode.ASYNC: AsyncExecutionStrategy(self)
+        }
+        
+        strategy: SyncExecutionStrategy | AsyncExecutionStrategy = execution_strategies[data.mode]
+        return strategy.execute(process_id, calculation_task)
+
+    def get_job_status(self, job_id: str) -> Dict[str, Any]:
+        """
+        Retrieves the status of a specific job.
+
+        Args:
+            job_id (str): The ID of the job.
+
+        Returns:
+            Dict[str, Any]: The status of the job.
+
+        Raises:
+            ValueError: If the job is not found.
+        """
+        # Check if job exists in Redis first
+        job_info = self.cache.get(f"job:{job_id}")
+        if not job_info:
+            logger.error(f"Job {job_id} not found in cache")
+            raise ValueError(f"Job {job_id} not found")
+
+        # Now check Celery status
+        result = AsyncResult(job_id)
+        if result.ready():
+            if result.successful():
+                return {"status": "successful", "type": "process"}
+            return {"status": "failed", "type": "process", "message": str(result.result)}
+        return {"status": "running", "type": "process"}
+
+    def get_job_result(self, job_id: str) -> Dict[str, Any]:
+        """
+        Retrieves the result of a specific job.
+
+        Args:
+            job_id (str): The ID of the job.
+
+        Returns:
+            Dict[str, Any]: The result of the job.
+
+        Raises:
+            ValueError: If the job is not found.
+        """
+        # Check if job exists in Redis first
+        job_info = self.cache.get(f"job:{job_id}")
+        if not job_info:
+            logger.error(f"Job {job_id} not found in cache")
+            raise ValueError(f"Job {job_id} not found")
+
+        result = AsyncResult(job_id)
+        if result.state == 'PENDING':
+            logger.error(f"Result for job ID {job_id} is not ready")
+            raise ValueError("Result not ready")
+        elif result.state == 'FAILURE':
+            logger.error(f"Job ID {job_id} failed with error: {result.result}")
+            raise ValueError(f"Job failed: {result.result}")
+        elif result.state == 'SUCCESS':
+            logger.info(f"Job ID {job_id} completed successfully")
+            return result.result
+        else:
+            return {"status": "running", "type": "process"}
+
+    def delete_job(self, job_id: str) -> Dict[str, Any]:
+        logger.info(f"Deleting job ID: {job_id}")
+        """
+        Deletes a specific job.
+
+        Args:
+            job_id (str): The ID of the job.
+
+        Returns:
+            Dict[str, Any]: The status of the deletion.
+
+        Raises:
+            ValueError: If the job is not found.
+        """
+        result = AsyncResult(job_id)
+        if not result:
+            logger.error("Job not found")
+            raise ValueError("Job not found")
+        result.forget()
+        return {"status": "dismissed", "message": "Job dismissed"}
+
+    def get_jobs(self, limit: int, offset: int) -> List[Dict[str, Any]]:
+        """
+        Retrieves a list of all jobs and their status.
+        
+        Returns:
+            List[Dict[str, Any]]: List of job status information
+        """
+        # Get all job IDs from Redis
+        job_keys = self.cache.keys("job:*")
+        jobs = []
+
+        for job_key in job_keys[offset:offset+limit]:
+            try:
+                job_info = self.cache.get(job_key)
+                if job_info:
+                    # Remove "job:" prefix for consistent job ID handling
+                    job_id = job_key.replace("job:", "")
+                    status_info = {
+                        "jobID": job_id,  # Clean job ID without prefix
+                        "status": job_info.get("status", "unknown"),
+                        "type": "process",
+                        "processID": job_info.get("process_id"),
+                        "created": job_info.get("created"),
+                        "started": job_info.get("started"),
+                        "finished": job_info.get("finished"),
+                        "updated": job_info.get("updated"),
+                        "progress": job_info.get("progress"),
+                        "links": [
+                            {
+                                "href": f"/jobs/{job_id}",  # Clean job ID in links
+                                "rel": "self",
+                                "type": "application/json"
+                            },
+                            {
+                                "href": f"/jobs/{job_id}/results",  # Clean job ID in links
+                                "rel": "results",
+                                "type": "application/json"
+                            }
+                        ]
+                    }
+                    if "message" in job_info:
+                        status_info["message"] = job_info["message"]
+                    jobs.append(status_info)
+            except Exception as e:
+                logger.error(f"Error retrieving job {job_key}: {e}")
+
+        next_link = None
+        if offset + limit < len(job_keys):
+            next_link = f"/jobs?limit={limit}&offset={offset+limit}"
+
+        return jobs, next_link
+
+    def _check_cache(self, calculation_task: CalculationTask) -> ProcessExecResponse | None:
+        """
+        Optimizes performance by checking if identical calculation exists in cache.
+        Uses task input hash as cache key.
+        
+        Args:
+            calculation_task: Task containing input parameters
+            
+        Returns:
+            Cached response if found, None otherwise
+        """
+        cache_check = self.celery_app.send_task(
+            'check_cache',
+            args=[calculation_task.model_dump()]
+        )
+        
+        cache_status = cache_check.get(timeout=10)
+        if cache_status["status"] != "HIT":
+            return None
+            
+        task = self.celery_app.send_task(
+            'find_result_in_cache',
+            args=[calculation_task.celery_key]
+        )
+        
+        job_info = {
+            "status": "successful",
+            "type": "process",
+            "created": datetime.utcnow().isoformat(),
+            "started": datetime.utcnow().isoformat(),
+            "finished": datetime.utcnow().isoformat(),
+            "updated": datetime.utcnow().isoformat(),
+            "progress": 100,
+            "message": "Result retrieved from cache"
+        }
+        self.cache.put(f"job:{task.id}", job_info)
+        
+        return ProcessExecResponse(status="successful", jobID=task.id, type="process")

fastprocesses/api/router.py

@@ -0,0 +1,177 @@
+from fastapi import APIRouter, HTTPException, Query, Response, status
+from fastapi.responses import JSONResponse
+
+from fastprocesses.api.manager import ProcessManager
+from fastprocesses.core.logging import logger
+from fastprocesses.core.models import (
+    Conformance,
+    ExecutionMode,
+    JobList,
+    Landing,
+    Link,
+    ProcessDescription,
+    ProcessExecRequestBody,
+    ProcessExecResponse,
+    ProcessList
+)
+
+
+def get_router(
+        process_manager: ProcessManager,
+        title: str,
+        description: str
+    ) -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/")
+    async def landing_page() -> Landing:
+        logger.debug("Landing page accessed")
+        return Landing(
+            title=title,
+            description=description,
+            links=[
+                Link(href="/", rel="self", type="application/json"),
+                Link(href="/conformance", rel="conformance", type="application/json"),
+                Link(href="/processes", rel="processes", type="application/json"),
+                Link(href="/jobs", rel="jobs", type="application/json"),
+            ]
+        )
+
+    @router.get("/conformance")
+    async def conformance() -> Conformance:
+        logger.debug("Conformance endpoint accessed")
+        return Conformance(
+            conformsTo=[
+                "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core",
+                "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json"
+                "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/job-list"
+            ]
+        )
+
+    @router.get(
+        "/processes",
+        response_model_exclude_none=True,
+        response_model=ProcessList
+    )
+    async def list_processes(
+        limit: int = Query(10, ge=1, le=10000),
+        offset: int = Query(0, ge=0)
+    ):
+        logger.debug("List processes endpoint accessed")
+
+        processes, next_link = process_manager.get_available_processes(limit, offset)
+        links = [Link(href="/processes", rel="self", type="application/json")]
+        if next_link:
+            links.append(Link(href=next_link, rel="next", type="application/json"))
+
+        return ProcessList(
+            processes=processes,
+            links=links
+        )
+
+    @router.get(
+            "/processes/{process_id}",
+            response_model_exclude_none=True,
+            response_model=ProcessDescription
+    )
+    async def describe_process(process_id: str):
+        logger.debug(f"Describe process endpoint accessed for process ID: {process_id}")
+        try:
+            return process_manager.get_process_description(process_id)
+        except ValueError as e:
+            logger.error(f"Process {process_id} not found: {e}")
+            raise HTTPException(status_code=404, detail=str(e))
+
+    @router.post(
+        "/processes/{process_id}/execution",
+        response_model=ProcessExecResponse
+    )
+    async def execute_process(
+        process_id: str,
+        request: ProcessExecRequestBody,
+        response: Response
+    ) -> JSONResponse:
+        logger.debug(f"Execute process endpoint accessed for process ID: {process_id}")
+        try:
+            result = process_manager.execute_process(process_id, request)
+            
+            # Set response status code based on execution mode
+            if request.mode == ExecutionMode.ASYNC:
+                response.status_code = status.HTTP_201_CREATED
+                # Add Location header for async execution
+                response.headers["Location"] = f"/jobs/{result.jobID}"
+            else:
+                # For sync execution with results
+                if result.value:
+                    response.status_code = status.HTTP_200_OK
+                # For sync execution without results
+                else:
+                    response.status_code = status.HTTP_204_NO_CONTENT
+            
+            return result
+        except ValueError as e:
+            error_message = str(e)
+            if "Input validation failed" in error_message:
+                logger.error(f"Input validation error for process {process_id}: {error_message}")
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail={
+                        "type": "process",
+                        "error": "InvalidParameterValue",
+                        "message": error_message,
+                        "process_id": process_id
+                    }
+                )
+            logger.error(f"Process {process_id} not found: {error_message}")
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail={
+                    "type": "process",
+                    "error": "NotFound",
+                    "message": error_message
+                }
+            )
+
+    @router.get(
+        "/jobs",
+        response_model_exclude_none=True,
+        response_model=JobList
+    )
+    async def list_jobs(
+        limit: int = Query(10, ge=1, le=1000),
+        offset: int = Query(0, ge=0)
+    ):
+        """
+        Lists all jobs.
+        """
+        logger.debug("List jobs endpoint accessed")
+        jobs, next_link = process_manager.get_jobs(limit, offset)
+        links = [Link(href="/jobs", rel="self", type="application/json")]
+        if next_link:
+            links.append(Link(href=next_link, rel="next", type="application/json"))
+
+        return JobList(
+            jobs=jobs,
+            links=links
+        )
+
+
+    @router.get("/jobs/{job_id}")
+    async def get_job_status(job_id: str):
+        logger.debug(f"Get job status endpoint accessed for job ID: {job_id}")
+        try:
+            return process_manager.get_job_status(job_id)
+        except ValueError as e:
+            logger.error(f"Job {job_id} not found: {e}")
+            raise HTTPException(status_code=404, detail=str(e))
+
+    @router.get("/jobs/{job_id}/results")
+    async def get_job_result(job_id: str):
+        logger.debug(f"Get job result endpoint accessed for job ID: {job_id}")
+        try:
+            return process_manager.get_job_result(job_id)
+        except ValueError as e:
+            logger.error(f"Job {job_id} not found: {e}")
+            raise HTTPException(status_code=404, detail=str(e))
+
+    return router

fastprocesses/api/server.py

@@ -0,0 +1,24 @@
+# src/fastprocesses/api/server.py
+from fastapi import FastAPI
+
+from fastprocesses.api.manager import ProcessManager
+from fastprocesses.api.router import get_router
+
+
+class OGCProcessesAPI:
+    def __init__(self, title: str, version: str, description: str):
+        self.process_manager = ProcessManager()
+        self.app = FastAPI(
+            title=title,
+            version=version,
+            description=description
+        )
+        self.app.include_router(
+            get_router(
+                self.process_manager,
+                self.app.title,
+                self.app.description)
+            )
+
+    def get_app(self) -> FastAPI:
+        return self.app

fastprocesses/celery_worker.py

@@ -0,0 +1,45 @@
+import signal
+import subprocess
+import sys
+
+from fastprocesses.core.logging import logger
+
+
+def main():
+    logger.info("Starting Celery worker")
+    # Construct the Celery command
+    celery_command = [
+        "celery",
+        "-A",
+        "src.fastprocesses.worker.celery_app",
+        "worker",
+        "--loglevel=info"
+    ]
+
+    # Append any additional arguments passed to the script
+    celery_command.extend(sys.argv[1:])
+
+    # Start the Celery process
+    process = subprocess.Popen(celery_command)
+    logger.info("Celery worker started")
+
+    def handle_signal(sig, frame):
+        logger.info(f"Received signal: {sig}")
+        # Forward the signal to the Celery process
+        process.send_signal(sig)
+
+    # Register signal handlers
+    signal.signal(signal.SIGINT, handle_signal)
+    signal.signal(signal.SIGTERM, handle_signal)
+
+    try:
+        # Wait for the Celery process to complete
+        process.wait()
+    except KeyboardInterrupt:
+        logger.info("KeyboardInterrupt received, shutting down Celery worker")
+        # Handle the KeyboardInterrupt to ensure a warm shutdown
+        process.send_signal(signal.SIGINT)
+        process.wait()
+
+if __name__ == "__main__":
+    main()

fastprocesses/common.py

@@ -0,0 +1,59 @@
+import json
+
+from celery import Celery
+from fastapi.encoders import jsonable_encoder
+from kombu.serialization import register
+
+from fastprocesses.core.cache import Cache
+from fastprocesses.core.config import settings
+
+
+def custom_json_serializer(obj):
+    # Use jsonable_encoder to handle complex objects
+    return json.dumps(jsonable_encoder(obj))
+
+
+def custom_json_deserializer(data):
+    # Deserialize JSON back into Python objects
+    return json.loads(data)
+
+# Register the custom serializer
+register(
+    "custom_json",
+    custom_json_serializer,
+    custom_json_deserializer,
+    content_type="application/x-custom-json",
+    content_encoding="utf-8",
+)
+
+celery_app = Celery(
+    "ogc_processes",
+    broker=str(settings.celery_broker_url),
+    backend=str(settings.celery_result_backend),
+    include=["fastprocesses.worker.celery_app"],  # Ensure the module is included
+)
+
+celery_app.conf.update(
+    task_serializer="custom_json",
+    result_serializer="custom_json",
+    accept_content=["custom_json", "json"],  # Accept only the custom serializer
+    timezone="UTC",
+    enable_utc=True,
+    task_routes={
+        "fastprocesses.worker.celery_app.execute_process": {
+            "queue": "process_tasks",
+            "routing_key": "process_tasks",
+        }
+    },
+    broker_connection_retry=True,
+    broker_connection_retry_on_startup=True,
+    # set limits for long-running tasks
+    task_time_limit=900,  # Hard limit in seconds
+    task_soft_time_limit=600,  # Soft limit in seconds
+    result_expires=3153600,  # 1 year in seconds
+    worker_send_task_events=True,  # Enable events to track task progress
+    # task_acks_late=True,  # Acknowledge the task only after it has been executed)
+
+)
+
+redis_cache = Cache(key_prefix="process_results", ttl_days=7)