fastprocesses 0.22.0__tar.gz → 0.22.2__tar.gz

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fastprocesses
-Version: 0.22.0
+Version: 0.22.2
 Summary: A library to create a FastAPI-based  OGC API Processes wrapper around existing projects.
 License-File: AUTHORS.md
 Author: Stefan Schuhart
@@ -39,7 +39,7 @@ A library to create a FastAPI-based OGC API Processes wrapper around existing pr
 
 AI helped to create this code.
 
-## Version: 0.22.0
+## Version: 0.22.2
 
 ### Description
 

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/README.md

@@ -4,7 +4,7 @@ A library to create a FastAPI-based OGC API Processes wrapper around existing pr
 
 AI helped to create this code.
 
-## Version: 0.22.0
+## Version: 0.22.2
 
 ### Description
 

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/pyproject.toml

@@ -79,7 +79,7 @@ exclude = [
 
 [tool.poetry]
 name = "fastprocesses"
-version = "0.22.0"
+version = "0.22.2"
 description = "A library to create a FastAPI-based  OGC API Processes wrapper around existing projects."
 authors = ["Stefan Schuhart <stefan.schuhart@gv.hamburg.de>"]
 readme = "README.md"

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/src/fastprocesses/core/base_process.py

@@ -104,7 +104,10 @@ class BaseProcess(ABC):
         else:
             return result
 
-    def resolve_remote_inputs(self, exec_body: Dict[str, Any]) -> Dict[str, Any]:
+    def resolve_remote_inputs(
+            self, exec_body: Dict[str, Any],
+            job_progress_callback: JobProgressCallback | None = None
+        ) -> Dict[str, Any]:
         """
         Substitutes URI-valued inputs with the downloaded data they reference.
 
@@ -238,7 +241,9 @@ class BaseProcess(ABC):
 
         return True
 
-    def validate_inputs(self, inputs: Dict[str, Any]) -> bool:
+    def validate_inputs(
+            self, inputs: Dict[str, Any],
+        ) -> bool:
         """
         Validates the input data against the process description.
 
@@ -464,7 +469,8 @@ class BaseParallelProcess(BaseProcess, ABC):
 
     @abstractmethod
     def merge_results(
-        self, results: List[Dict[str, Any]]
+        self, results: List[Dict[str, Any]],
+        job_progress_callback: JobProgressCallback | None = None,
     ) -> BaseProcessResult:
         """
         Combine N partial results into the final output.
@@ -610,6 +616,7 @@ class BaseScatterProcess(BaseProcess, ABC):
         self,
         results: Dict[str, Any],
         exec_body: Dict[str, Any],
+        job_progress_callback: JobProgressCallback | None = None,
     ) -> BaseProcessResult:
         """
         Combine the results of all parallel steps into the final output.

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/src/fastprocesses/worker/celery_app.py

@@ -116,7 +116,9 @@ def execute_process(self, process_id: str, serialized_data: str | bytes):
         logger.debug("Cache lookup failed (non-fatal), proceeding: {}", e)
 
     process = _load_process(process_id, job_id)
-    data = _run_pipeline(process, process_id, data, job_id)
+    data = _run_pipeline(
+        process, process_id, data, job_id, job_progress_callback
+    )
 
     try:
         update_job_status(

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/src/fastprocesses/worker/chord_tasks.py

@@ -17,7 +17,10 @@ from fastprocesses.core.models import CalculationTask, JobStatusCode
 from fastprocesses.processes.process_registry import get_process_registry
 from fastprocesses.worker.job_status import (
     _cleanup_progress_counter,
+    _cleanup_scatter_roster,
     _increment_and_report_progress,
+    _init_scatter_roster,
+    _record_scatter_step_done,
     update_job_status,
 )
 
@@ -270,7 +273,7 @@ def execute_scatter_step(
         result = jsonable_encoder(partial)
         rkey = _result_key(job_id, step_name)
         temp_result_cache.put(key=rkey, value=result)
-        _increment_and_report_progress(job_id, total)
+        _record_scatter_step_done(job_id, step_name, total)
         return {"__claim_check__": rkey}
     except Exception as exc:
         update_job_status(
@@ -310,6 +313,7 @@ def _run_scatter(
         f"{total} step(s): {step_names}."
     )
 
+    _init_scatter_roster(job_id, step_names)
     pkey = _payload_key(job_id)
     temp_result_cache.put(
         key=pkey,
@@ -357,7 +361,12 @@ def finalize_scatter(
 
         payload = temp_result_cache.get(payload_key)
         temp_result_cache.delete(payload_key)
-        exec_body: dict = payload["original_input"] if payload else {}
+        # step_input is the resolved data (after resolve_remote_inputs + pipeline).
+        # original_input is the wire-format exec_body (may contain URL strings)
+        # and is used only for deriving the stable cache key — never for business
+        # logic, because the resolved content may change between cache fills.
+        exec_body: dict = payload["step_input"] if payload else {}
+        original_input: dict = payload.get("original_input", exec_body) if payload else {}
 
         update_job_status(
             job_id, 95, "Merging scatter results.", JobStatusCode.RUNNING
@@ -366,7 +375,7 @@ def finalize_scatter(
         merged = merge_result.model_dump(mode="json")
 
         try:
-            calculation_task = CalculationTask(**exec_body)
+            calculation_task = CalculationTask(**original_input)
             temp_result_cache.put(key=calculation_task.celery_key, value=merged)
             # Also store under job_id so get_job_result can retrieve it.
             temp_result_cache.put(key=job_id, value=merged)
@@ -382,6 +391,7 @@ def finalize_scatter(
             job_id, 100, "Process completed.", JobStatusCode.SUCCESSFUL
         )
         _cleanup_progress_counter(job_id)
+        _cleanup_scatter_roster(job_id)
         logger.info(
             f"Scatter process {process_id} (job {job_id}) completed successfully."
         )

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/src/fastprocesses/worker/job_status.py

@@ -1,10 +1,12 @@
 from datetime import datetime, timezone
-
+import json
 from fastprocesses.common import job_status_cache
 from fastprocesses.core.logging import logger
 from fastprocesses.core.models import JobStatusCode, JobStatusInfo, Link
 
 _SUBTASK_PROGRESS_KEY = "fp:subtask_progress"
+_SCATTER_ALL_KEY = "fp:scatter_all"
+_SCATTER_DONE_KEY = "fp:scatter_done"
 
 
 def update_job_status(
@@ -144,3 +146,82 @@ def _cleanup_progress_counter(job_id: str) -> None:
         logger.warning(
             "Could not clean up progress counter for job {}: {!r}", job_id, exc
         )
+
+
+def _init_scatter_roster(job_id: str, step_names: list[str]) -> None:
+    """
+    Stores the full list of scatter step names for *job_id* so that any
+    subtask can later read it back to build the step-status message.
+    Called once by ``_run_scatter`` before the chord is dispatched.
+    """
+    try:
+        redis = job_status_cache.redis_connection
+        all_key = f"{_SCATTER_ALL_KEY}:{job_id}"
+        
+        redis._execute_redis_command("set", all_key, json.dumps(step_names))
+        redis._execute_redis_command("expire", all_key, 86400)
+    except Exception as exc:  # pragma: no cover
+        logger.warning(
+            "Could not initialise scatter roster for job {}: {!r}", job_id, exc
+        )
+
+
+def _record_scatter_step_done(job_id: str, step_name: str, total: int) -> None:
+    """
+    Marks *step_name* as completed for *job_id* and updates the job status
+    message with a human-readable roster:
+
+        Steps: vulnerability_pipeline ✓, exposure_pipeline ✓,
+               hazard_wb_pipeline …, hazard_ma_pipeline … (2/4 done)
+
+    Uses an atomic INCR for the progress percentage (same 0-90 % band as
+    ``_increment_and_report_progress``) and a Redis set for the done-step
+    roster so concurrent workers never race.
+
+    All exceptions are swallowed so a Redis hiccup never causes a subtask
+    to fail.
+    """
+    import json as json
+    counter_key = f"{_SUBTASK_PROGRESS_KEY}:{job_id}"
+    done_key = f"{_SCATTER_DONE_KEY}:{job_id}"
+    all_key = f"{_SCATTER_ALL_KEY}:{job_id}"
+    try:
+        redis = job_status_cache.redis_connection
+        completed = redis._execute_redis_command("incr", counter_key)
+        if completed == 1:
+            redis._execute_redis_command("expire", counter_key, 86400)
+        redis._execute_redis_command("sadd", done_key, step_name)
+        redis._execute_redis_command("expire", done_key, 86400)
+
+        # Read back all_steps and done_steps to build the message.
+        raw_all = redis._execute_redis_command("get", all_key)
+        all_steps: list[str] = json.loads(raw_all) if raw_all else []
+        done_raw = redis._execute_redis_command("smembers", done_key)
+        done_steps: set[str] = {
+            m.decode() if isinstance(m, bytes) else m for m in (done_raw or [])
+        }
+
+        parts = [
+            f"{s} \u2713" if s in done_steps else f"{s} \u2026"
+            for s in (all_steps or [step_name])
+        ]
+        message = ", ".join(parts) + f" ({len(done_steps)}/{total} done)"
+
+        progress = min(int(completed / total * 90), 90)
+        update_job_status(job_id, progress, message, JobStatusCode.RUNNING)
+    except Exception as exc:  # pragma: no cover
+        logger.warning(
+            "Could not record scatter step progress for job {}: {!r}", job_id, exc
+        )
+
+
+def _cleanup_scatter_roster(job_id: str) -> None:
+    """Deletes scatter roster keys for *job_id* after the job completes."""
+    try:
+        redis = job_status_cache.redis_connection
+        redis._execute_redis_command("delete", f"{_SCATTER_ALL_KEY}:{job_id}")
+        redis._execute_redis_command("delete", f"{_SCATTER_DONE_KEY}:{job_id}")
+    except Exception as exc:  # pragma: no cover
+        logger.warning(
+            "Could not clean up scatter roster for job {}: {!r}", job_id, exc
+        )

{fastprocesses-0.22.0 → fastprocesses-0.22.2}/src/fastprocesses/worker/pipeline.py

@@ -8,6 +8,7 @@ from fastprocesses.core.exceptions import (
 )
 from fastprocesses.core.logging import logger
 from fastprocesses.core.models import JobStatusCode
+from fastprocesses.core.types import JobProgressCallback
 from fastprocesses.processes.process_registry import get_process_registry
 from fastprocesses.worker.job_status import update_job_status
 
@@ -38,7 +39,8 @@ def _load_process(process_id: str, job_id: str) -> BaseProcess:
 
 
 def _run_pipeline(
-    process: BaseProcess, process_id: str, data: dict, job_id: str
+    process: BaseProcess, process_id: str, data: dict, job_id: str,
+    job_progress_callback: JobProgressCallback | None = None,
 ) -> dict:
     """
     Runs the pre-execution pipeline: validate → resolve remote inputs → late_validate.
@@ -47,7 +49,11 @@ def _run_pipeline(
     Returns the (possibly modified) data dict after remote input resolution.
     """
     # Step 1: validate wire-format inputs against the process description schema
-    update_job_status(job_id, 0, "Validating inputs.", JobStatusCode.RUNNING)
+    update_job_status(
+        job_id, 0,
+        "Pre-process step 1: Validating inputs against process description.",
+        JobStatusCode.RUNNING
+    )
     try:
         process.validate_inputs(data["inputs"])
     except ValueError as e:
@@ -56,9 +62,13 @@ def _run_pipeline(
         raise InputValidationError(process_id, repr(e))
 
     # Step 2: resolve remote inputs (URI strings → downloaded data)
-    update_job_status(job_id, 0, "Resolving remote inputs.", JobStatusCode.RUNNING)
+    update_job_status(
+        job_id, 0,
+        "Pre-process step 2: Resolving remote inputs.",
+        JobStatusCode.RUNNING
+    )
     try:
-        data = process.resolve_remote_inputs(data)
+        data = process.resolve_remote_inputs(data, job_progress_callback)
     except (SSRFBlockedError, ValueError) as e:
         logger.error(
             "Remote input resolution failed for process {}: {}", process_id, e
@@ -67,6 +77,11 @@ def _run_pipeline(
         raise InputValidationError(process_id, repr(e))
 
     # Step 3: late validation of resolved inputs (no-op by default)
+    update_job_status(
+        job_id, 0,
+        "Pre-process step 3: Validating remotely resolved inputs.",
+        JobStatusCode.RUNNING
+    )
     try:
         process.late_validate(data["inputs"])
     except ValueError as e: