fastprocesses 0.7.4__py3-none-any.whl

fastprocesses/worker/celery_app.py

@@ -0,0 +1,154 @@
+# worker/celery_app.py
+import asyncio
+from datetime import datetime, timezone
+import json
+from typing import Any, Dict
+
+from celery import Task
+from pydantic import BaseModel
+
+from fastprocesses.common import celery_app, redis_cache
+from fastprocesses.core.logging import logger
+from fastprocesses.core.models import CalculationTask
+from fastprocesses.processes.process_registry import get_process_registry
+
+from celery.exceptions import SoftTimeLimitExceeded
+
+# NOTE: Cache hash key is based on original unprocessed inputs always
+# this ensures consistent caching and cache retrieval
+# which does not depend on arbitrary processed data, which can change
+# when the process is updated or changed!
+
+class CacheResultTask(Task):
+    def on_success(
+            self, retval: dict | BaseModel,
+            task_id, args, kwargs
+    ):
+        try:
+            # Deserialize the original data
+            original_data = json.loads(args[1])
+            calculation_task = CalculationTask(**original_data)
+            
+            # Get the the hash key for the task
+            key = calculation_task.celery_key
+
+            # Store the result in cache
+            # Use the task ID as the key
+            redis_cache.put(key=key, value=retval)
+
+            logger.info(f"Saved result with key {key} to cache: {retval}")
+        except Exception as e:
+            logger.error(f"Error caching results: {e}")
+
+
+@celery_app.task(bind=True, name="execute_process", base=CacheResultTask)
+def execute_process(self, process_id: str, serialized_data: Dict[str, Any]):
+
+    data = json.loads(serialized_data)
+
+    logger.info(f"Executing process {process_id} with data {data}")
+    job_id = self.request.id  # Get the task/job ID
+
+    # Create a progress update function that captures the job_id
+    def update_progress(progress: int, message: str = None):
+        job_key = f"job:{job_id}"
+        job_info = redis_cache.get(job_key) or {}
+
+        job_info.update(
+            {
+                "status": "running",
+                "progress": progress,
+                "updated": datetime.now(timezone.utc),
+                "process_id": process_id,
+            }
+        )
+
+        if message:
+            job_info["message"] = message
+
+        redis_cache.put(job_key, job_info)
+        logger.debug(f"Updated progress for job {job_id}: {progress}%, {message}")
+
+    try:
+        # Initialize progress
+        update_progress(0, "Starting process")
+
+        service = get_process_registry().get_service(process_id)
+
+        if asyncio.iscoroutinefunction(service.execute):
+            result = asyncio.run(
+                service.execute(
+                    data,
+                    progress_callback=update_progress,
+                )
+            )
+        else:
+            result = service.execute(data)
+
+        logger.info(f"Process {process_id} executed successfully with result {result}")
+
+        # Mark job as complete
+        update_progress(100, "Process completed")
+
+        return result
+    except SoftTimeLimitExceeded as e:
+        logger.warning(f"Task {job_id} hit the soft time limit: {e}")
+        # Attempt to resume the process
+        try:
+            if asyncio.iscoroutinefunction(service.execute):
+                result = asyncio.run(service.execute(data))
+            else:
+                result = service.execute(data)
+
+            logger.info(f"Process {process_id} completed after soft time limit")
+            update_progress(100, "Process completed after soft time limit")
+            return result
+        except Exception as inner_exception:
+            logger.error(
+                f"Error while completing task after soft time limit: {inner_exception}"
+            )
+            raise inner_exception
+
+    except Exception as e:
+        # Update job with error status
+        job_key = f"job:{job_id}"
+        job_info = redis_cache.get(job_key) or {}
+        job_info.update(
+            {
+                "status": "failed",
+                "message": str(e),
+                "updated": datetime.now(timezone.utc),
+                "process_id": process_id,
+            }
+        )
+        redis_cache.put(job_key, job_info)
+
+        logger.error(f"Error executing process {process_id}: {e}")
+        raise
+
+
+@celery_app.task(name="check_cache")
+def check_cache(calculation_task: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Check if results exist in cache and return status
+    """
+    task = CalculationTask(**calculation_task)
+    cached_result = redis_cache.get(key=task.celery_key)
+
+    if cached_result:
+        logger.info(f"Cache hit for key {task.celery_key}")
+        return {"status": "HIT", "result": cached_result}
+
+    logger.info(f"Cache miss for key {task.celery_key}")
+    return {"status": "MISS"}
+
+
+@celery_app.task(name="find_result_in_cache")
+def find_result_in_cache(celery_key: str) -> dict | None:
+    """
+    Retrieve result from cache
+    """
+    result = redis_cache.get(key=celery_key)
+    if result:
+        logger.info(f"Retrieved result from cache for key {celery_key}")
+    return result

fastprocesses-0.7.4.dist-info/AUTHORS.md

@@ -0,0 +1 @@
+# Authors

fastprocesses-0.7.4.dist-info/METADATA

@@ -0,0 +1,274 @@
+Metadata-Version: 2.3
+Name: fastprocesses
+Version: 0.7.4
+Summary: A library to create a FastAPI-based  OGC API Processes wrapper around existing projects.
+Author: Stefan Schuhart
+Author-email: stefan.schuhart@gv.hamburg.de
+Requires-Python: >=3.10
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development
+Classifier: Topic :: Utilities
+Classifier: Typing :: Typed
+Requires-Dist: celery (>=5.4.0,<6.0.0)
+Requires-Dist: fastapi (>=0.115.8,<0.116.0)
+Requires-Dist: loguru (>=0.7.3,<0.8.0)
+Requires-Dist: pydantic (>=2.10.6,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.7.1,<3.0.0)
+Requires-Dist: redis (>=5.2.1,<6.0.0)
+Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
+Project-URL: Changelog, https://github.com/geowerkstatt-hamburg/fastProcesses/CHANGELOG.md
+Project-URL: Documentation, https://github.com/geowerkstatt-hamburg/fastProcesses/README.md
+Project-URL: Homepage, https://geowerkstatt-hamburg.github.io/fastProcesses
+Project-URL: Repository, https://github.com/geowerkstatt-hamburg/fastProcesses
+Description-Content-Type: text/markdown
+
+# fastprocesses
+
+A library to create a FastAPI-based OGC API Processes wrapper around existing projects. This library simplifies the process of defining and registering processes, making it easy to build and deploy OGC API Processes.
+
+AI was used to create this code.
+
+## Version: 0.7.4
+
+### Description
+
+fastprocesses is a Python library that provides a simple and efficient way to create OGC API Processes using FastAPI. It allows you to define processes, register them, and expose them through a FastAPI application with minimal effort, following the OGC API Processes 1.0.0 specification.
+
+### Features
+
+- **OGC API Processes Compliance**: Fully implements the OGC API Processes 1.0.0 Core specification
+- **FastAPI Integration**: Leverages FastAPI for building high-performance APIs
+- **Process Management**: Supports both synchronous and asynchronous process execution
+- **Job Control**: Implements job control options (sync-execute, async-execute)
+- **Output Handling**: Supports various output transmission modes (value, reference)
+- **Result Caching**: Built-in Redis-based caching for process results
+- **Celery Integration**: Asynchronous task processing using Celery
+- **Pydantic Models**: Strong type validation for process inputs and outputs
+- **Logging**: Uses `loguru` for modern logging with rotation support
+
+### Architecture
+
+```mermaid
+graph TB
+    subgraph Client
+        CLI[Client Request]
+    end
+
+    subgraph FastAPI Application
+        API[OGCProcessesAPI]
+        Router[API Router]
+        PM[ProcessManager]
+        PR[ProcessRegistry]
+    end
+
+    subgraph Redis
+        RC[Redis Cache]
+        RR[Redis Registry]
+    end
+
+    subgraph Process
+        BP[BaseProcess]
+        SP[SimpleProcess]
+    end
+
+    subgraph Worker
+        CW[Celery Worker]
+        CT[CacheResultTask]
+    end
+
+    %% Client interactions
+    CLI -->|HTTP Request| API
+    API -->|Route Request| Router
+    Router -->|Execute Process| PM
+
+    %% Process Manager flow
+    PM -->|Get Process| PR
+    PM -->|Check Cache| RC
+    PM -->|Submit Task| CW
+    PM -->|Get Result| RC
+
+    %% Process Registry
+    PR -->|Store/Retrieve| RR
+    PR -.->|Registers| SP
+    SP -->|Inherits| BP
+
+    %% Worker flow
+    CW -->|Execute| SP
+    CW -->|Cache Result| CT
+    CT -->|Store| RC
+
+    %% Styling
+    classDef api fill:#f9f,stroke:#333,stroke-width:2px
+    classDef cache fill:#bbf,stroke:#333,stroke-width:2px
+    classDef process fill:#bfb,stroke:#333,stroke-width:2px
+    classDef worker fill:#fbb,stroke:#333,stroke-width:2px
+
+    class API,Router api
+    class RC,RR cache
+    class BP,SP process
+    class CW,CT worker
+```
+
+### Usage
+
+1. **Define a Process**: Create a new process by subclassing `BaseProcess` and using the `@register_process` decorator.
+
+```python
+from fastprocesses.core.base_process import BaseProcess
+from fastprocesses.core.models import (
+    ProcessDescription,
+    ProcessInput,
+    ProcessJobControlOptions,
+    ProcessOutput,
+    ProcessOutputTransmission,
+    Schema,
+)
+from fastprocesses.processes.process_registry import register_process
+
+@register_process("simple_process")
+class SimpleProcess(BaseProcess):
+    # Define process description as a class variable
+    process_description = ProcessDescription(
+        id="simple_process",
+        title="Simple Process",
+        version="1.0.0",
+        description="A simple example process",
+        jobControlOptions=[
+            ProcessJobControlOptions.SYNC_EXECUTE,
+            ProcessJobControlOptions.ASYNC_EXECUTE
+        ],
+        outputTransmission=[
+            ProcessOutputTransmission.VALUE
+        ],
+        inputs={
+            "input_text": ProcessInput(
+                title="Input Text",
+                description="Text to process",
+                schema=Schema(
+                    type="string",
+                    minLength=1,
+                    maxLength=1000
+                )
+            )
+        },
+        outputs={
+            "output_text": ProcessOutput(
+                title="Output Text",
+                description="Processed text",
+                schema=Schema(
+                    type="string"
+                )
+            )
+        },
+        keywords=["text", "processing"],
+        metadata={
+            "created": "2024-02-19",
+            "provider": "Example Organization"
+        }
+    )
+
+    async def execute(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        input_text = inputs["inputs"]["input_text"]
+        output_text = input_text.upper()
+        return {"output_text": output_text}
+```
+
+2. **Create the FastAPI Application**:
+
+```python
+import uvicorn
+from fastprocesses.api.server import OGCProcessesAPI
+
+app = OGCProcessesAPI(
+    title="Simple Process API",
+    version="1.0.0",
+    description="A simple API for running processes"
+).get_app()
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+```
+
+3. **Start the Services**:
+
+Start Redis (required for caching and Celery):
+```bash
+docker run -d -p 6379:6379 redis
+```
+
+Start the Celery worker:
+```bash
+celery -A fastprocesses.worker.celery_app worker
+```
+
+Start the FastAPI application:
+```bash
+poetry run python examples/run_example.py
+```
+
+4. **Use the API**:
+
+Execute a process (async):
+```bash
+curl -X POST "http://localhost:8000/processes/simple_process/execution" \
+     -H "Content-Type: application/json" \
+     -d '{
+       "inputs": {
+         "input_text": "hello world"
+       },
+       "mode": "async"
+     }'
+```
+
+Execute a process (sync):
+```bash
+curl -X POST "http://localhost:8000/processes/simple_process/execution" \
+     -H "Content-Type: application/json" \
+     -d '{
+       "inputs": {
+         "input_text": "hello world"
+       },
+       "mode": "sync"
+     }'
+```
+
+### API Endpoints
+
+- `GET /`: Landing page
+- `GET /conformance`: OGC API conformance declaration
+- `GET /processes`: List available processes
+- `GET /processes/{process_id}`: Get process description
+- `POST /processes/{process_id}/execution`: Execute a process
+- `GET /jobs`: List all jobs
+- `GET /jobs/{job_id}`: Get job status
+- `GET /jobs/{job_id}/results`: Get job results
+
+### Configuration
+
+The library can be configured using environment variables:
+
+```bash
+REDIS_CACHE_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/1
+CELERY_RESULT_BACKEND=redis://localhost:6379/2
+```
+
+### Notes:
+How to serialize pydantic models within celery? -> https://benninger.ca/posts/celery-serializer-pydantic/
+
+!IMPORTANT!: Cache hash key is based on original unprocessed inputs always this ensures consistent caching and cache retrieval which does not depend on arbitrary processed data, which can change when the process is updated or changed!
+
+### Version Notes
+- **Version: 0.7.4**: added paging to processes and jobs, including limit and offset query params
+- **0.5.0**: Extended Schema model
+- **0.4.0**: Added full OGC API Processes 1.0.0 Core compliance
+- **0.3.0**: Added job control and output transmission options
+- **0.2.0**: Added Redis caching and Celery integration
+- **0.1.0**: Initial release with basic process support

fastprocesses-0.7.4.dist-info/RECORD

@@ -0,0 +1,23 @@
+fastprocesses/__init__.py,sha256=OG7vLPTeVifB2d4TZn55ZGUwSnPvzGIjjgwXtOh69KA,181
+fastprocesses/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fastprocesses/api/manager.py,sha256=wDxWd8nwu4xvOf4tJzyqFP6wiFrK4JbC3CjIydoQsm0,13850
+fastprocesses/api/router.py,sha256=uymoKUoqCRtvPhK-qI1VJJZIVdyJEMhc0T8zDtSa5ks,6265
+fastprocesses/api/server.py,sha256=9egolkONiMYDGlRsAFCPB3Zf2ayCjeS-XVdLAll77w8,671
+fastprocesses/celery_worker.py,sha256=2BF7pey9nVrRffEqZJD2jCqNMxP8qsCLQejFng537wc,1211
+fastprocesses/common.py,sha256=sZn8B6RfsvG7xwgMnsq8_svyGBOrndHgG6pW9Ozfmq0,1777
+fastprocesses/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fastprocesses/core/base_process.py,sha256=sAj_RfG3q9Pv2c7hhKu66NH4oIMlwhHhA_Hu8FH0MJI,5099
+fastprocesses/core/cache.py,sha256=hZzJcpCYqP94hlllv5cyYTIZS0-aXq5kxHrXBDasoRc,1932
+fastprocesses/core/config.py,sha256=I9hIKbGXUWEkSgDSS9KuGAZ4pEu9fwTE3AczRmdNct4,569
+fastprocesses/core/logging.py,sha256=GpCQn5bvq-hVxtYMzVCYoHvZF37hiI2q4Kpj7vys9Io,1816
+fastprocesses/core/models.py,sha256=67SVAQvh-K-tjlt_t79PcPc7ZJ4YYUGzRnoliqcHi2M,4796
+fastprocesses/processes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fastprocesses/processes/process_registry.py,sha256=ugsXD0Rj5-Snj--e1FeIzg9CVKhRFlllJVXupBcURr0,4267
+fastprocesses/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fastprocesses/worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fastprocesses/worker/celery_app.py,sha256=KEuulMqUp29zgD_XB5fsD49eI4qcU96YfWVpqxQ8Z5c,5058
+fastprocesses-0.7.4.dist-info/AUTHORS.md,sha256=F8zDIs87H_lW-w4iCvDHMfX84oMW6x4AGtK1Rd4hGBY,9
+fastprocesses-0.7.4.dist-info/METADATA,sha256=cf7MfKSgPNWk-OlkspYE3Wiu0ShEjkAF42A-oHpch_s,8478
+fastprocesses-0.7.4.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+fastprocesses-0.7.4.dist-info/entry_points.txt,sha256=MeSG-KBsVp1wDIvyaatauFal78lgb-0VJUFRsPZ5pkw,72
+fastprocesses-0.7.4.dist-info/RECORD,,

fastprocesses-0.7.4.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.1
+Root-Is-Purelib: true
+Tag: py3-none-any

fastprocesses-0.7.4.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+start-celery-worker=fastprocesses.celery_worker:main
+