fastprocesses 0.19.2__tar.gz → 0.19.4__tar.gz

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fastprocesses
-Version: 0.19.2
+Version: 0.19.4
 Summary: A library to create a FastAPI-based  OGC API Processes wrapper around existing projects.
 Author: Stefan Schuhart
 Author-email: stefan.schuhart@gv.hamburg.de
@@ -38,7 +38,7 @@ A library to create a FastAPI-based OGC API Processes wrapper around existing pr
 
 AI helped to create this code.
 
-## Version: 0.19.2
+## Version: 0.19.4
 
 ### Description
 

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/README.md

@@ -4,7 +4,7 @@ A library to create a FastAPI-based OGC API Processes wrapper around existing pr
 
 AI helped to create this code.
 
-## Version: 0.19.2
+## Version: 0.19.4
 
 ### Description
 

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/pyproject.toml

@@ -79,7 +79,7 @@ exclude = [
 
 [tool.poetry]
 name = "fastprocesses"
-version = "0.19.2"
+version = "0.19.4"
 description = "A library to create a FastAPI-based  OGC API Processes wrapper around existing projects."
 authors = ["Stefan Schuhart <stefan.schuhart@gv.hamburg.de>"]
 readme = "README.md"

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/src/fastprocesses/api/manager.py

@@ -94,6 +94,12 @@ class AsyncExecutionStrategy(ExecutionStrategy):
             )
             raise BrokerUnavailableError(str(exc)) from exc
         send_elapsed = time.monotonic() - send_start
+        logger.info(
+            "Task queued: process_id={}, job_id={}, mode=async, send_time={:.3f}s",
+            process_id,
+            task.id,
+            send_elapsed,
+        )
         if send_elapsed > 1.0:
             logger.warning(
                 "Celery async send_task for process_id={} took {:.2f}s",
@@ -155,6 +161,13 @@ class AsyncExecutionStrategy(ExecutionStrategy):
             )
             raise BrokerUnavailableError(str(exc)) from exc
         send_elapsed = time.monotonic() - send_start
+        logger.info(
+            "Task queued: process_id={}, job_id={},"
+            " mode=async (raw), send_time={:.3f}s",
+            process_id,
+            task.id,
+            send_elapsed,
+        )
         if send_elapsed > 1.0:
             logger.warning(
                 "Celery async send_task for process_id={} took {:.2f}s",
@@ -218,6 +231,13 @@ class SyncExecutionStrategy(ExecutionStrategy):
                 exc,
             )
             raise BrokerUnavailableError(str(exc)) from exc
+        send_elapsed_sync = time.monotonic() - send_start
+        logger.info(
+            "Task queued: process_id={}, job_id={}, mode=sync, send_time={:.3f}s",
+            process_id,
+            task.id,
+            send_elapsed_sync,
+        )
         send_elapsed = time.monotonic() - send_start
         if send_elapsed > 1.0:
             logger.warning(

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/src/fastprocesses/common.py

@@ -65,6 +65,7 @@ celery_app = Celery(
 )
 
 celery_app.conf.update(
+    task_default_queue=settings.FP_CELERY_QUEUE,
     task_serializer="custom_json",
     result_serializer="custom_json",
     accept_content=["custom_json", "json"],  # Accept only the custom serializer

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/src/fastprocesses/core/base_process.py

@@ -5,6 +5,7 @@ from typing import Any, Awaitable, ClassVar, Dict, List
 
 from jsonschema import ValidationError as JSONSchemaValidationError
 from jsonschema import validate as jsonschema_validate
+from jsonschema.exceptions import best_match as _jsonschema_best_match
 from pydantic import BaseModel
 from referencing import Registry
 from referencing.exceptions import Unresolvable as _UnresolvableRef
@@ -13,6 +14,18 @@ from fastprocesses.core.logging import logger
 from fastprocesses.core.models import OutputControl, ProcessDescription
 from fastprocesses.core.types import JobProgressCallback
 
+# Maps JSON Schema primitive type names to their Python equivalents.
+# Used by quick_validate_inputs for a cheap top-level type check.
+_JS_TYPE_TO_PYTHON: Dict[str, Any] = {
+    "string": str,
+    "number": (int, float),
+    "integer": int,
+    "boolean": bool,
+    "array": list,
+    "object": dict,
+    "null": type(None),
+}
+
 
 class BaseProcess(ABC):
     process_description: ClassVar[ProcessDescription]
@@ -98,9 +111,10 @@ class BaseProcess(ABC):
         unchanged, so existing processes are unaffected.
 
         Override this in subclasses whose inputs may be supplied as URIs
-        instead of inline data.  The worker calls this method *before*
-        ``validate_inputs``, so the resolved data is what gets validated
-        and passed to ``execute``.
+        instead of inline data.  The worker calls this method *after*
+        ``validate_inputs`` so that the process description schema is validated
+        against the wire-format input (the URI reference object), and *before*
+        ``late_validate`` which validates the resolved data.
 
         The HTTP fetch, authentication, redirect policy, and SSRF checks are
         the responsibility of the overriding application — fastprocesses does
@@ -128,21 +142,99 @@ class BaseProcess(ABC):
         """
         return exec_body
 
+    def late_validate(self, inputs: Dict[str, Any]) -> bool:
+        """
+        Process-specific validation of **resolved** inputs, called after
+        :meth:`resolve_remote_inputs` and before ``execute``.
+
+        The default implementation is a **no-op** that always returns ``True``,
+        so existing processes are unaffected.
+
+        Override this in subclasses that need to validate data that was
+        fetched or transformed by ``resolve_remote_inputs`` — for example,
+        validating a downloaded GeoJSON FeatureCollection against a
+        process-specific Pydantic model, or spot-checking the first feature
+        to detect structural problems early.
+
+        Unlike :meth:`validate_inputs`, which uses the generic process
+        description schema, ``late_validate`` has full access to the resolved
+        data and can apply arbitrary business logic.  Raise ``ValueError`` to
+        fail the job with a user-facing message.
+
+        Example::
+
+            from mypackage.models import BuildingFeatureCollection
+
+            def late_validate(self, inputs: dict) -> bool:
+                # Validate the fetched collection against a Pydantic model.
+                # model_validate raises ValidationError on structural problems.
+                BuildingFeatureCollection.model_validate(
+                    inputs["buildings"]
+                )
+                return True
+        """
+        return True
+
     def quick_validate_inputs(self, inputs: Dict[str, Any]) -> bool:
         """
-        Quickly checks that all required input fields are present.
-        Does NOT perform deep schema validation.
+        Cheap structural check run at the API boundary (before the job is queued).
+
+        Catches the three most common user errors without any recursion:
+        1. Unknown field names (e.g. a typo like ``"buldings"`` instead of
+           ``"buildings"``)
+        2. Missing required fields
+        3. Top-level type mismatch (e.g. passing a string where an object is
+           expected)
+
+        Deep schema validation (nested properties, ``oneOf``, ``enum``,
+        ``const``, …) is left to :meth:`validate_inputs` which runs on the
+        worker after the job has been accepted.
         """
         description: ProcessDescription = self.get_description()
-        required_inputs = description.inputs
+        defined_inputs = description.inputs
 
-        # Check for missing required inputs only
-        for input_name, input_desc in required_inputs.items():
-            if input_desc.minOccurs > 0 and input_name not in inputs:
+        # 1. Unknown fields — catches typos before a job is even queued
+        unknown = sorted(k for k in inputs if k not in defined_inputs)
+        if unknown:
+            raise ValueError(
+                f"Unknown input(s): {', '.join(unknown)}. "
+                f"Expected: {', '.join(sorted(defined_inputs))}."
+            )
+
+        # 2. Missing required fields
+        for name, desc in defined_inputs.items():
+            if desc.minOccurs > 0 and name not in inputs:
+                raise ValueError(
+                    f"Missing required input '{name}'. "
+                    f"Description: {desc.description}"
+                )
+
+        # 3. Top-level type check — O(n inputs), no recursion into nested data
+        for name, value in inputs.items():
+            schema_type = defined_inputs[name].scheme.type
+            if schema_type is None:
+                continue  # no top-level type declared (e.g. pure oneOf/allOf)
+            declared = [schema_type] if isinstance(schema_type, str) else schema_type
+            expected = tuple(
+                py_t
+                for js_t in declared
+                if (py_t := _JS_TYPE_TO_PYTHON.get(js_t)) is not None
+            )
+            if not expected:
+                continue
+            # bool is a subclass of int; reject it unless "boolean" is declared
+            if isinstance(value, bool) and "boolean" not in declared:
                 raise ValueError(
-                    f"Missing required input '{input_name}'. "
-                    f"Description: {input_desc.description}"
+                    f"Input '{name}' has wrong type: "
+                    f"expected {', '.join(declared)}, got boolean."
                 )
+            if not isinstance(value, expected):
+                raise ValueError(
+                    f"Input '{name}' has wrong type: "
+                    f"expected {', '.join(declared)}, "
+                    f"got {type(value).__name__}."
+                )
+
         return True
 
     def validate_inputs(self, inputs: Dict[str, Any]) -> bool:
@@ -163,6 +255,8 @@ class BaseProcess(ABC):
 
         # First, check all provided inputs
         for input_name, input_value in inputs.items():
+            logger.info("Validating input '{}'", input_name)
+
             if input_name not in required_inputs:
                 raise ValueError(
                     f"Provided input '{input_name}' is "
@@ -179,9 +273,23 @@ class BaseProcess(ABC):
                     registry=self.schema_registry,
                 )
             except JSONSchemaValidationError as e:
+                # For oneOf/anyOf/allOf, best_match picks the deepest,
+                # most specific sub-error instead of the generic top-level one.
+                leaf = _jsonschema_best_match(e.context) if e.context else e
+                # Build a readable path: features[42].properties.area_m2
+                path_parts: list[str] = []
+                for step in leaf.absolute_path:
+                    if isinstance(step, int):
+                        if path_parts:
+                            path_parts[-1] += f"[{step}]"
+                        else:
+                            path_parts.append(f"[{step}]")
+                    else:
+                        path_parts.append(str(step))
+                location = ".".join(path_parts) or "(root)"
                 raise ValueError(
-                    f"Input '{input_name}' validation failed: {e.message}. "
-                    f"Description: {input_desc.scheme.model_dump(exclude_unset=True, by_alias=True)}"
+                    f"Input '{input_name}' validation failed: "
+                    f"{leaf.message} (at: {location})."
                 )
             except Exception as e:
                 # jsonschema wraps referencing.exceptions.Unresolvable as
@@ -203,14 +311,6 @@ class BaseProcess(ABC):
                 else:
                     raise
 
-        # Then, check for missing required inputs
-        for input_name, input_desc in required_inputs.items():
-            if input_desc.minOccurs > 0 and input_name not in inputs:
-                raise ValueError(
-                    f"Missing required input '{input_name}'. "
-                    f"Description: {input_desc.description}"
-                )
-
         return True
 
     def validate_outputs(

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/src/fastprocesses/core/config.py

@@ -74,6 +74,10 @@ class OGCProcessesSettings(BaseSettings):
     FP_CELERY_RESULTS_TTL_DAYS: int = 365
     FP_CELERY_TASK_TLIMIT_HARD: int = 900 # seconds
     FP_CELERY_TASK_TLIMIT_SOFT: int = 600 # seconds
+    FP_CELERY_QUEUE: str = Field(
+        default="celery",
+        description="Celery queue name for task routing. Must match the worker -Q flag.",
+    )
     FP_CELERY_JOB_MODE: bool = Field(
         default=False,
         description="Enable job mode for graceful shutdown after task completion"

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/src/fastprocesses/processes/process_registry.py

@@ -15,7 +15,7 @@ class ProcessRegistry:
     """Manages the registration and retrieval of available processs (processes)."""
 
     def __init__(self, redis_connection: RedisConnection | None = None):
-        self.registry_key = "process_registry"
+        self.registry_key = f"process_registry:{settings.FP_CELERY_QUEUE}"
         if redis_connection is None:
             redis_connection = RedisConnection(str(settings.results_cache.connection))
         self.redis_connection = redis_connection

{fastprocesses-0.19.2 → fastprocesses-0.19.4}/src/fastprocesses/worker/celery_app.py

@@ -3,6 +3,8 @@ import asyncio
 import inspect
 import json
 import signal
+import time
+import traceback
 from datetime import datetime, timezone
 from typing import Any, Dict, cast
 
@@ -581,12 +583,16 @@ def execute_process(self, process_id: str, serialized_data: str | bytes):
         serialized_data_str = bytes(serialized_data).decode("utf-8")
     data: dict = json.loads(serialized_data_str)
 
-    logger.info(f"Executing process {process_id} with data {serialized_data[:80]}")
     job_id = self.request.id  # Get the task/job ID
+    task_start = time.monotonic()
+    logger.info(
+        "Worker picked up task: process_id={}, job_id={}",
+        process_id,
+        job_id,
+    )
 
     # First: Get the process
     try:
-        logger.info(f"Worker retrieving process {process_id} from registry")
         process = get_process_registry().get_process(process_id)
     except ValueError as e:
         job_status = JobStatusCode.FAILED
@@ -600,7 +606,27 @@ def execute_process(self, process_id: str, serialized_data: str | bytes):
     except ProcessClassNotFoundError as e:
         raise e
 
-    # Second: resolve remote inputs (URI strings → downloaded data)
+    # Second: validate wire-format inputs against the process description schema
+    try:
+        update_job_status(
+            job_id,
+            0,
+            "Validating inputs.",
+            job_status,
+        )
+        process.validate_inputs(data["inputs"])
+    except ValueError as e:
+        logger.error(f"Input validation failed for process {process_id}: {str(e)}")
+        job_status = JobStatusCode.FAILED
+        update_job_status(
+            job_id,
+            0,
+            str(e),
+            job_status,
+        )
+        raise InputValidationError(process_id, repr(e))
+
+    # Third: resolve remote inputs (URI strings → downloaded data)
     try:
         update_job_status(
             job_id,
@@ -617,30 +643,19 @@ def execute_process(self, process_id: str, serialized_data: str | bytes):
         update_job_status(job_id, 0, str(e), job_status)
         raise InputValidationError(process_id, repr(e))
 
-    # Third: deep validation of inputs
+    # Fourth: late validation of resolved inputs (process-specific, no-op by default)
     try:
-        logger.info(f"Worker validating inputs for process {process_id}")
-        update_job_status(
-            job_id,
-            0,
-            "Validating inputs. This may take a while for complex inputs.",
-            job_status,
-        )
-        process.validate_inputs(data["inputs"])
+        process.late_validate(data["inputs"])
     except ValueError as e:
-        logger.error(f"Input validation failed for process {process_id}: {str(e)}")
-        job_status = JobStatusCode.FAILED
-        update_job_status(
-            job_id,
-            0,
-            str(e),
-            job_status,
+        logger.error(
+            "Late validation failed for process {}: {}", process_id, e
         )
+        job_status = JobStatusCode.FAILED
+        update_job_status(job_id, 0, str(e), job_status)
         raise InputValidationError(process_id, repr(e))
 
     # Fourth: Execute the process
     try:
-        logger.info("Worker executing process {}", process_id)
         job_status = JobStatusCode.RUNNING
         # BUG: if redis returns no job_status, this fails and creates a ValueError too
         update_job_status(
@@ -690,21 +705,33 @@ def execute_process(self, process_id: str, serialized_data: str | bytes):
     except Exception as e:
         # Update job with error status
         job_status = JobStatusCode.FAILED
+        task_elapsed = time.monotonic() - task_start
 
         # decide if its a validation error or a general error
         if isinstance(e, ValueError) or isinstance(e, ValidationError):
-            logger.error(f"Validation error in process {process_id}: {e}")
+            logger.error(
+                "Task failed: process_id={}, job_id={}, duration={:.2f}s, reason={}",
+                process_id,
+                job_id,
+                task_elapsed,
+                e,
+            )
             job_message = e
             raise e
 
         # Log the full traceback server-side only — never expose internal
         # class names, source lines, or line numbers to the client.
-        logger.exception(
-            "Unhandled exception in process '{}' (job {}): {}",
+        logger.error(
+            "Task failed: process_id={}, job_id={}, duration={:.2f}s, reason={}",
             process_id,
             job_id,
+            task_elapsed,
             e,
-            exc_info=True,
+        )
+        logger.debug(
+            "Full traceback for job {}:\n{}",
+            job_id,
+            traceback.format_exc(),
         )
 
         job_message = (
@@ -720,10 +747,13 @@ def execute_process(self, process_id: str, serialized_data: str | bytes):
             return None
 
         if result:
-            result_dump = jsonable_encoder(result)
+            task_elapsed = time.monotonic() - task_start
             logger.info(
-                f"Process {process_id} executed "
-                f"successfully with result {json.dumps(result_dump)[:80]}"
+                "Task completed: process_id={}, job_id={},"
+                " duration={:.2f}s",
+                process_id,
+                job_id,
+                task_elapsed,
             )
             job_status = JobStatusCode.SUCCESSFUL