fastprocesses 0.7.4__py3-none-any.whl

fastprocesses/core/base_process.py

@@ -0,0 +1,137 @@
+from abc import ABC, abstractmethod
+from typing import Any, Callable, ClassVar, Dict, List
+
+from pydantic import BaseModel
+
+from fastprocesses.core.models import ProcessDescription
+
+
+class BaseProcess(ABC):
+    process_description: ClassVar[ProcessDescription]
+
+    def get_description(self) -> ProcessDescription:
+        """
+        Returns the OGC API Process description.
+        
+        Returns:
+            ProcessDescription: Complete process description following OGC API standard
+        """
+        if not hasattr(self, 'process_description'):
+            raise NotImplementedError(
+                f"Process class {self.__class__.__name__} must define 'process_description'"
+            )
+        return self.process_description
+
+    @classmethod
+    def create_description(cls, description_dict: Dict[str, Any]) -> ProcessDescription:
+        """
+        Creates a ProcessDescription from a dictionary.
+        
+        Args:
+            description_dict (Dict[str, Any]): Dictionary containing process description
+            
+        Returns:
+            ProcessDescription: Validated process description object
+        """
+        return ProcessDescription.model_validate(description_dict)
+
+    @abstractmethod
+    async def execute(
+        self,
+        exec_body: Dict[str, Any],
+        progress_callback: Callable[[int, str], None] | None = None
+    ) -> Dict[str, Any]:
+        """
+        Executes the process with given inputs.
+        
+        Args:
+            inputs (Dict[str, Any]): Input parameters matching the process description
+            
+        Returns:
+            Dict[str, Any]: Output values matching the process description
+            
+        Raises:
+            ValueError: If inputs are invalid
+        """
+        pass
+
+    def validate_inputs(self, inputs: Dict[str, Any]) -> bool:
+        """
+        Validates the input data against the process description.
+        
+        Args:
+            inputs (Dict[str, Any]): The input data to validate
+            
+        Returns:
+            bool: True if inputs are valid
+            
+        Raises:
+            ValueError: With detailed error message if validation fails
+        """
+        description = self.get_description()
+        required_inputs = description.inputs
+        
+        # Check for missing required inputs
+        for input_name, input_desc in required_inputs.items():
+            if input_desc.minOccurs > 0 and input_name not in inputs:
+                raise ValueError(
+                    f"Missing required input '{input_name}'. "
+                    f"Description: {input_desc.get('description', 'No description available')}"
+                )
+            
+            # Validate input type if schema is provided
+            if input_name in inputs:
+                expected_type = input_desc.scheme.type
+                if expected_type == "string" and not isinstance(inputs[input_name], str):
+                    raise ValueError(
+                        f"Invalid type for input '{input_name}'. "
+                        f"Expected string, got {type(inputs[input_name]).__name__}. "
+                        f"Description: {input_desc.get('description', 'No description available')}"
+                    )
+                elif expected_type == "number" and not isinstance(inputs[input_name], (int, float)):
+                    raise ValueError(
+                        f"Invalid type for input '{input_name}'. "
+                        f"Expected number, got {type(inputs[input_name]).__name__}. "
+                        f"Description: {input_desc.get('description', 'No description available')}"
+                    )
+                # Add more type validations as needed
+    
+        return True
+    
+    def validate_outputs(self, outputs: str | List[str]) -> bool:
+        """
+        Validates the outputs parameter against the process description.
+        
+        Args:
+            outputs: Single output identifier or list of output identifiers
+                
+        Returns:
+            bool: True if outputs are valid
+                
+        Raises:
+            ValueError: If any output identifier is invalid
+        """
+        description = self.get_description()
+        available_outputs = description.outputs.keys()
+        
+        if not available_outputs:
+            raise ValueError("Process has no defined outputs")
+        
+        # Convert single string to list for uniform handling
+        output_list = [outputs] if isinstance(outputs, str) else outputs
+        
+        if not output_list:
+            # If no outputs specified, all outputs are considered valid
+            return True
+        
+        # Validate each output identifier
+        invalid_outputs = [out for out in output_list if out not in available_outputs]
+        if invalid_outputs:
+            available = ", ".join(available_outputs)
+            invalid = ", ".join(invalid_outputs)
+            raise ValueError(
+                f"Invalid output identifiers: {invalid}. "
+                f"Available outputs are: {available}"
+            )
+        
+        return True

fastprocesses/core/cache.py

@@ -0,0 +1,55 @@
+import json
+
+import redis
+from fastapi.encoders import jsonable_encoder
+
+from fastprocesses.core.config import settings
+from fastprocesses.core.logging import logger
+
+
+class Cache:
+    def __init__(self, key_prefix: str, ttl_days: int):
+        self._redis = redis.Redis.from_url(str(settings.redis_cache_url), decode_responses=True)
+        self._key_prefix = key_prefix
+        self._ttl_days = ttl_days
+
+    def get(self, key: str) -> dict:
+        logger.debug(f"Getting cache for key: {key}")
+        key = self._make_key(key)
+        serialized_value = self._redis.get(key)
+        
+        logger.debug(f"{serialized_value}")
+        return None if serialized_value is None else json.loads(serialized_value)
+
+    def put(self, key: str, value: dict) -> None:
+        logger.debug(f"Putting cache for key: {key}")
+        key = self._make_key(key)
+        jsonable_value = jsonable_encoder(value)
+        serialized_value = json.dumps(jsonable_value)
+        ttl = self._ttl_days * 86400
+        self._redis.setex(key, ttl, serialized_value)
+
+    def delete(self, key: str) -> None:
+        logger.debug(f"Deleting cache for key: {key}")
+        key = self._make_key(key)
+        self._redis.delete(key)
+
+    def _make_key(self, key: str) -> str:
+        return f"{self._key_prefix}:{key}"
+
+    def keys(self, pattern: str = "*") -> list[str]:
+        """
+        Get all keys matching the pattern.
+        
+        Args:
+            pattern (str): Redis key pattern to match. Defaults to "*".
+            
+        Returns:
+            list[str]: List of matching keys without the prefix
+        """
+        logger.debug(f"Getting keys matching pattern: {pattern}")
+        full_pattern = self._make_key(pattern)
+        keys = self._redis.keys(full_pattern)
+        # Remove prefix from keys before returning
+        prefix_len = len(self._key_prefix) + 1  # +1 for the colon
+        return [key[prefix_len:] for key in keys]

fastprocesses/core/config.py

@@ -0,0 +1,17 @@
+from pydantic import RedisDsn, AnyUrl
+from pydantic_settings import BaseSettings
+
+
+class OGCProcessesSettings(BaseSettings):
+    api_title: str = "Simple Process API"
+    api_version: str = "1.0.0"
+    api_description: str = "A simple API for running processes"
+    celery_broker_url: RedisDsn = RedisDsn("redis://redis:6379/0")
+    celery_result_backend: RedisDsn = RedisDsn("redis://redis:6379/0")
+    redis_cache_url: RedisDsn = RedisDsn("redis://redis:6379/1")
+
+    class Config:
+        env_file = ".env"
+        extra = "ignore"
+
+settings = OGCProcessesSettings()

fastprocesses/core/logging.py

@@ -0,0 +1,75 @@
+import logging
+import sys
+
+from loguru import logger
+
+# Remove existing handlers
+for handler in logging.root.handlers[:]:
+    logging.root.removeHandler(handler)
+
+loggers = (
+    "uvicorn",
+    "uvicorn.access",
+    "uvicorn.error",
+    "fastapi",
+    "asyncio",
+    "starlette",
+)
+
+for logger_name in loggers:
+    logging_logger = logging.getLogger(logger_name)
+    logging_logger.handlers = []
+    logging_logger.propagate = True
+
+class InterceptHandler(logging.Handler):
+    def emit(self, record):
+        # Get corresponding Loguru level
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        # Find caller to get correct stack depth
+        frame, depth = logging.currentframe(), 2
+        while frame.f_back and frame.f_code.co_filename == logging.__file__:
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(
+            level, record.getMessage()
+        )
+
+# Configure logger
+logger.remove()  # Remove default logger
+
+# Error logs to stderr
+logger.add(
+    sys.stderr, 
+    level="ERROR", 
+    format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}"
+)
+
+# Error logs to file with weekly rotation
+logger.add(
+    "logs/errors_{time}.log", 
+    level="ERROR", 
+    rotation="1 week", 
+    retention="1 month", 
+    format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}"
+)
+
+# Info logs to stdout
+# TODO read loglevel from settings!
+logger.add(
+    sys.stdout, 
+    level="DEBUG", 
+    format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}",
+    backtrace=True,
+    diagnose=True,
+)
+
+# Intercept standard logging
+logging.basicConfig(handlers=[InterceptHandler()], level=logging.INFO)
+
+# Example usage
+logger.info("Logging setup complete.")

fastprocesses/core/models.py

@@ -0,0 +1,188 @@
+import hashlib
+import json
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
+
+from fastapi.encoders import jsonable_encoder
+from pydantic import AfterValidator, BaseModel, ConfigDict, Field, computed_field, field_validator
+
+
+class Link(BaseModel):
+    href: str
+    rel: str
+    type: str
+
+
+class Landing(BaseModel):
+    title: str
+    description: str
+    links: List[Link]
+
+
+class Conformance(BaseModel):
+    conformsTo: List[str]
+
+
+class ProcessJobControlOptions(str, Enum):
+    SYNC_EXECUTE = "sync-execute"
+    ASYNC_EXECUTE = "async-execute"
+    DISMISS = "dismiss"
+
+
+# TODO: needs to be passed to outputs keys and when part of the data validated
+# TODO: transmission mode can be different for each output
+# https://schemas.opengis.net/ogcapi/processes/part1/1.0/examples/json/ProcessDescription.json
+class ProcessOutputTransmission(str, Enum):
+    VALUE = "value"
+    REFERENCE = "reference"
+
+
+class ResponseType(str, Enum):
+    RAW = "raw"
+    DOCUMENT = "document"
+
+
+class Schema(BaseModel):
+    type: Optional[str] = None
+    format: Optional[str] = None
+    minimum: Optional[float] = None
+    maximum: Optional[float] = None
+    minLength: Optional[int] = None
+    maxLength: Optional[int] = None
+    pattern: Optional[str] = None
+    enum: Optional[List[Any]] = None
+    properties: Optional[Dict[str, Any]] = None
+    required: Optional[List[str]] = None
+    items: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None
+    oneOf: Optional[List[Dict[str, Any]]] = None
+    allOf: Optional[List[Dict[str, Any]]] = None
+    contentMediaType: Optional[str] = None
+    contentEncoding: Optional[str] = None
+    contentSchema: Optional[str] = None
+
+    class Config:
+        exclude_none = True
+
+
+class ProcessInput(BaseModel):
+    title: str
+    description: str
+    scheme: Schema = Field(alias="schema")
+    minOccurs: Optional[int] = 1
+    maxOccurs: Optional[int] = 1
+    metadata: Optional[Dict[str, Any]] = None
+
+    class Config:
+        exclude_none = True
+        populate_by_name = True
+
+
+class ProcessOutput(BaseModel):
+    title: str
+    description: str
+    scheme: Schema = Field(alias="schema")
+    metadata: Optional[Dict[str, Any]] = None
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        exclude_none=True,
+    )
+
+
+class ProcessDescription(BaseModel):
+    id: str
+    title: str
+    description: str
+    version: str
+    jobControlOptions: List[ProcessJobControlOptions]
+    outputTransmission: List[ProcessOutputTransmission]
+    inputs: Dict[str, ProcessInput]
+    outputs: Dict[str, ProcessOutput]
+    links: Optional[List[Link]] = None
+    keywords: Optional[List[str]] = None
+    metadata: Optional[Dict[str, Any]] = None
+
+    class Config:
+        exclude_none = True
+        populate_by_name = True
+
+
+class ProcessList(BaseModel):
+    processes: List[ProcessDescription]
+    links: Optional[List[Link]] = None
+
+
+class ExecutionMode(str, Enum):
+    SYNC = "sync"
+    ASYNC = "async"
+
+
+class OutputControl(BaseModel):
+    transmissionMode: Literal["value", "reference"] | None = "value"
+    format: dict | None = None
+
+
+class ProcessExecRequestBody(BaseModel):
+    inputs: Dict[str, Any]
+    outputs: dict[str, dict[str, OutputControl]] | None = None
+    mode: Optional[ExecutionMode] = ExecutionMode.ASYNC
+    response: Optional[ResponseType] = ResponseType.RAW
+
+    model_config = ConfigDict(exclude_none=True)
+
+
+def deserialize_json(value: Any) -> Any:
+    return jsonable_encoder(value)
+
+
+class CalculationTask(BaseModel):
+    inputs: Annotated[Dict[str, Any], AfterValidator(deserialize_json)]
+    outputs: dict[str, dict[str, OutputControl]] | None = None
+    response: ResponseType = ResponseType.RAW
+
+    def _hash_dict(self):
+        return hashlib.sha256(
+            json.dumps(self.inputs, sort_keys=True).encode()
+        ).hexdigest()
+
+    @computed_field
+    def celery_key(self) -> str:
+        return self._hash_dict()
+
+
+class ProcessExecResponse(BaseModel):
+    status: str
+    jobID: str
+    type: str = "process"
+
+
+class ProcessSummary(BaseModel):
+    """
+    The OGC conform ProcessSummary Model.
+    """
+
+    id: str
+    version: str
+    links: Optional[list[Link]] = None
+
+
+class JobStatusInfo(BaseModel):
+    jobID: str
+    status: str
+    type: str = "process"
+    processID: Optional[str] = None
+    message: Optional[str] = None
+    created: Optional[datetime] = None
+    started: Optional[datetime] = None
+    finished: Optional[datetime] = None
+    updated: Optional[datetime] = None
+    progress: Optional[int] = Field(None, ge=0, le=100)
+    links: Optional[List[Link]] = None
+
+    model_config = ConfigDict(populate_by_name=True, exclude_none=True)
+
+
+class JobList(BaseModel):
+    jobs: List[JobStatusInfo]
+    links: List[Link]

fastprocesses/processes/process_registry.py

@@ -0,0 +1,114 @@
+# src/fastprocesses/services/service_registry.py
+import json
+from pydoc import locate
+from typing import List
+
+import redis
+
+from fastprocesses.core.base_process import BaseProcess
+from fastprocesses.core.config import settings
+from fastprocesses.core.logging import logger
+from fastprocesses.core.models import ProcessDescription
+
+
+class ProcessRegistry:
+    """Manages the registration and retrieval of available services (processes)."""
+
+    def __init__(self):
+        """Initializes the ProcessRegistry with Redis connection."""
+        self.redis = redis.Redis.from_url(str(settings.redis_cache_url))
+        self.registry_key = "service_registry"
+
+    def register_service(self, process_id: str, service: BaseProcess):
+        """
+        Registers a process service in Redis:
+        - Stores process description and class path for dynamic loading
+        - Uses Redis hash structure for efficient lookups
+        - Enables service discovery and instantiation
+        """
+        try:
+            description: ProcessDescription = service.get_description()
+            
+            # serialize the description
+            description_dict = description.model_dump(exclude_none=True)
+            service_data = {
+                "description": description_dict,
+                "class_path": f"{service.__module__}.{service.__class__.__name__}"
+            }
+            logger.debug(f"Service data to be registered: {service_data}")
+            self.redis.hset(self.registry_key, process_id, json.dumps(service_data))
+        except Exception as e:
+            logger.error(f"Failed to register service {process_id}: {e}")
+            raise
+
+    def get_service_ids(self) -> List[str]:
+        """
+        Retrieves the IDs of all registered services.
+
+        Returns:
+            List[str]: A list of service IDs.
+        """
+        logger.debug("Retrieving all registered service IDs")
+        return [key.decode('utf-8') for key in self.redis.hkeys(self.registry_key)]
+
+    def has_service(self, process_id: str) -> bool:
+        """
+        Checks if a service is registered.
+
+        Args:
+            process_id (str): The ID of the process.
+
+        Returns:
+            bool: True if the service is registered, False otherwise.
+        """
+        logger.debug(f"Checking if service with ID {process_id} is registered")
+        return self.redis.hexists(self.registry_key, process_id)
+
+    def get_service(self, process_id: str) -> BaseProcess:
+        """
+        Dynamically loads and instantiates a process service:
+        1. Retrieves service metadata from Redis
+        2. Uses Python's module system to locate the class
+        3. Instantiates a new service instance
+        
+        The locate() function dynamically imports the class based on its path.
+        """
+        logger.info(f"Retrieving service with ID: {process_id}")
+        service_data = self.redis.hget(self.registry_key, process_id)
+        
+        if not service_data:
+            logger.error(f"Service {process_id} not found!")
+            raise ValueError(f"Service {process_id} not found!")
+            
+        service_info = json.loads(service_data)
+        service_class = locate(service_info["class_path"])
+        
+        if not service_class:
+            logger.error(f"Service class {service_info['class_path']} not found!")
+            # raise ValueError(f"Service class {service_info['class_path']} not found!")
+            
+        return service_class()
+
+# Global instance of ProcessRegistry
+_global_process_registry = ProcessRegistry()
+
+def get_process_registry() -> ProcessRegistry:
+    """Returns the global ProcessRegistry instance."""
+    return _global_process_registry
+
+def register_process(process_id: str):
+    """
+    Decorator for automatic process registration.
+    Allows processes to self-register by simply using @register_process decorator.
+    Example:
+        @register_process("my_process")
+        class MyProcess(BaseProcess):
+            ...
+    """
+    def decorator(cls):
+        if not hasattr(cls, 'process_description'):
+            raise ValueError(f"Process {cls.__name__} must define a 'description' class variable")
+        get_process_registry().register_service(process_id, cls())
+        return cls
+    return decorator
+