fastprocesses 0.22.1__tar.gz → 0.22.2__tar.gz

{fastprocesses-0.22.1 → fastprocesses-0.22.2}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: fastprocesses
-Version: 0.22.1
+Version: 0.22.2
 Summary: A library to create a FastAPI-based  OGC API Processes wrapper around existing projects.
+License-File: AUTHORS.md
 Author: Stefan Schuhart
 Author-email: stefan.schuhart@gv.hamburg.de
 Requires-Python: >=3.12
@@ -38,7 +39,7 @@ A library to create a FastAPI-based OGC API Processes wrapper around existing pr
 
 AI helped to create this code.
 
-## Version: 0.22.1
+## Version: 0.22.2
 
 ### Description
 

{fastprocesses-0.22.1 → fastprocesses-0.22.2}/README.md

@@ -4,7 +4,7 @@ A library to create a FastAPI-based OGC API Processes wrapper around existing pr
 
 AI helped to create this code.
 
-## Version: 0.22.1
+## Version: 0.22.2
 
 ### Description
 

{fastprocesses-0.22.1 → fastprocesses-0.22.2}/pyproject.toml

@@ -79,7 +79,7 @@ exclude = [
 
 [tool.poetry]
 name = "fastprocesses"
-version = "0.22.1"
+version = "0.22.2"
 description = "A library to create a FastAPI-based  OGC API Processes wrapper around existing projects."
 authors = ["Stefan Schuhart <stefan.schuhart@gv.hamburg.de>"]
 readme = "README.md"

{fastprocesses-0.22.1 → fastprocesses-0.22.2}/src/fastprocesses/worker/chord_tasks.py

@@ -17,7 +17,10 @@ from fastprocesses.core.models import CalculationTask, JobStatusCode
 from fastprocesses.processes.process_registry import get_process_registry
 from fastprocesses.worker.job_status import (
     _cleanup_progress_counter,
+    _cleanup_scatter_roster,
     _increment_and_report_progress,
+    _init_scatter_roster,
+    _record_scatter_step_done,
     update_job_status,
 )
 
@@ -270,7 +273,7 @@ def execute_scatter_step(
         result = jsonable_encoder(partial)
         rkey = _result_key(job_id, step_name)
         temp_result_cache.put(key=rkey, value=result)
-        _increment_and_report_progress(job_id, total)
+        _record_scatter_step_done(job_id, step_name, total)
         return {"__claim_check__": rkey}
     except Exception as exc:
         update_job_status(
@@ -310,6 +313,7 @@ def _run_scatter(
         f"{total} step(s): {step_names}."
     )
 
+    _init_scatter_roster(job_id, step_names)
     pkey = _payload_key(job_id)
     temp_result_cache.put(
         key=pkey,
@@ -357,7 +361,12 @@ def finalize_scatter(
 
         payload = temp_result_cache.get(payload_key)
         temp_result_cache.delete(payload_key)
-        exec_body: dict = payload["original_input"] if payload else {}
+        # step_input is the resolved data (after resolve_remote_inputs + pipeline).
+        # original_input is the wire-format exec_body (may contain URL strings)
+        # and is used only for deriving the stable cache key — never for business
+        # logic, because the resolved content may change between cache fills.
+        exec_body: dict = payload["step_input"] if payload else {}
+        original_input: dict = payload.get("original_input", exec_body) if payload else {}
 
         update_job_status(
             job_id, 95, "Merging scatter results.", JobStatusCode.RUNNING
@@ -366,7 +375,7 @@ def finalize_scatter(
         merged = merge_result.model_dump(mode="json")
 
         try:
-            calculation_task = CalculationTask(**exec_body)
+            calculation_task = CalculationTask(**original_input)
             temp_result_cache.put(key=calculation_task.celery_key, value=merged)
             # Also store under job_id so get_job_result can retrieve it.
             temp_result_cache.put(key=job_id, value=merged)
@@ -382,6 +391,7 @@ def finalize_scatter(
             job_id, 100, "Process completed.", JobStatusCode.SUCCESSFUL
         )
         _cleanup_progress_counter(job_id)
+        _cleanup_scatter_roster(job_id)
         logger.info(
             f"Scatter process {process_id} (job {job_id}) completed successfully."
         )

{fastprocesses-0.22.1 → fastprocesses-0.22.2}/src/fastprocesses/worker/job_status.py

@@ -1,10 +1,12 @@
 from datetime import datetime, timezone
-
+import json
 from fastprocesses.common import job_status_cache
 from fastprocesses.core.logging import logger
 from fastprocesses.core.models import JobStatusCode, JobStatusInfo, Link
 
 _SUBTASK_PROGRESS_KEY = "fp:subtask_progress"
+_SCATTER_ALL_KEY = "fp:scatter_all"
+_SCATTER_DONE_KEY = "fp:scatter_done"
 
 
 def update_job_status(
@@ -144,3 +146,82 @@ def _cleanup_progress_counter(job_id: str) -> None:
         logger.warning(
             "Could not clean up progress counter for job {}: {!r}", job_id, exc
         )
+
+
+def _init_scatter_roster(job_id: str, step_names: list[str]) -> None:
+    """
+    Stores the full list of scatter step names for *job_id* so that any
+    subtask can later read it back to build the step-status message.
+    Called once by ``_run_scatter`` before the chord is dispatched.
+    """
+    try:
+        redis = job_status_cache.redis_connection
+        all_key = f"{_SCATTER_ALL_KEY}:{job_id}"
+        
+        redis._execute_redis_command("set", all_key, json.dumps(step_names))
+        redis._execute_redis_command("expire", all_key, 86400)
+    except Exception as exc:  # pragma: no cover
+        logger.warning(
+            "Could not initialise scatter roster for job {}: {!r}", job_id, exc
+        )
+
+
+def _record_scatter_step_done(job_id: str, step_name: str, total: int) -> None:
+    """
+    Marks *step_name* as completed for *job_id* and updates the job status
+    message with a human-readable roster:
+
+        Steps: vulnerability_pipeline ✓, exposure_pipeline ✓,
+               hazard_wb_pipeline …, hazard_ma_pipeline … (2/4 done)
+
+    Uses an atomic INCR for the progress percentage (same 0-90 % band as
+    ``_increment_and_report_progress``) and a Redis set for the done-step
+    roster so concurrent workers never race.
+
+    All exceptions are swallowed so a Redis hiccup never causes a subtask
+    to fail.
+    """
+    import json as json
+    counter_key = f"{_SUBTASK_PROGRESS_KEY}:{job_id}"
+    done_key = f"{_SCATTER_DONE_KEY}:{job_id}"
+    all_key = f"{_SCATTER_ALL_KEY}:{job_id}"
+    try:
+        redis = job_status_cache.redis_connection
+        completed = redis._execute_redis_command("incr", counter_key)
+        if completed == 1:
+            redis._execute_redis_command("expire", counter_key, 86400)
+        redis._execute_redis_command("sadd", done_key, step_name)
+        redis._execute_redis_command("expire", done_key, 86400)
+
+        # Read back all_steps and done_steps to build the message.
+        raw_all = redis._execute_redis_command("get", all_key)
+        all_steps: list[str] = json.loads(raw_all) if raw_all else []
+        done_raw = redis._execute_redis_command("smembers", done_key)
+        done_steps: set[str] = {
+            m.decode() if isinstance(m, bytes) else m for m in (done_raw or [])
+        }
+
+        parts = [
+            f"{s} \u2713" if s in done_steps else f"{s} \u2026"
+            for s in (all_steps or [step_name])
+        ]
+        message = ", ".join(parts) + f" ({len(done_steps)}/{total} done)"
+
+        progress = min(int(completed / total * 90), 90)
+        update_job_status(job_id, progress, message, JobStatusCode.RUNNING)
+    except Exception as exc:  # pragma: no cover
+        logger.warning(
+            "Could not record scatter step progress for job {}: {!r}", job_id, exc
+        )
+
+
+def _cleanup_scatter_roster(job_id: str) -> None:
+    """Deletes scatter roster keys for *job_id* after the job completes."""
+    try:
+        redis = job_status_cache.redis_connection
+        redis._execute_redis_command("delete", f"{_SCATTER_ALL_KEY}:{job_id}")
+        redis._execute_redis_command("delete", f"{_SCATTER_DONE_KEY}:{job_id}")
+    except Exception as exc:  # pragma: no cover
+        logger.warning(
+            "Could not clean up scatter roster for job {}: {!r}", job_id, exc
+        )