howler-api 3.4.0.dev929__tar.gz → 3.4.0.dev933__tar.gz

{howler_api-3.4.0.dev929 → howler_api-3.4.0.dev933}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 3.4.0.dev929
+Version: 3.4.0.dev933
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-3.4.0.dev929 → howler_api-3.4.0.dev933}/howler/api/v1/hit.py

@@ -123,7 +123,7 @@ def create_hits(user: User, **kwargs):
 
             datastore().hit.commit()
 
-            action_service.bulk_execute_on_query(f"howler.id:({' OR '.join(odm.howler.id for odm in odms)})", user=user)
+            action_service.enqueue_action_execution([odm.howler.id for odm in odms], trigger="create", user=user)
 
         response_body["warnings"] = warnings
 
@@ -551,11 +551,7 @@ def add_label(id, label_set, user, **kwargs):
 
     datastore().hit.commit()
 
-    action_service.bulk_execute_on_query(
-        f"howler.id:{id}",
-        trigger="add_label",
-        user=user,
-    )
+    action_service.enqueue_action_execution([id], trigger="add_label", user=user)
 
     hit, version = hit_service.get_hit(id, as_odm=False, version=True)
 
@@ -609,11 +605,7 @@ def remove_labels(id, label_set, user, **kwargs):
 
     datastore().hit.commit()
 
-    action_service.bulk_execute_on_query(
-        f"howler.id:{id}",
-        trigger="remove_label",
-        user=user,
-    )
+    action_service.enqueue_action_execution([id], trigger="remove_label", user=user)
 
     hit, version = hit_service.get_hit(id, as_odm=False, version=True)
 

{howler_api-3.4.0.dev929 → howler_api-3.4.0.dev933}/howler/api/v1/tool.py

@@ -178,6 +178,6 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
 
         datastore().hit.commit()
 
-        action_service.bulk_execute_on_query(f"howler.id:({' OR '.join(entry['id'] for entry in out)})", user=user)
+        action_service.enqueue_action_execution([entry["id"] for entry in out], trigger="create", user=user)
 
         return created(out, warnings=warnings)

howler_api-3.4.0.dev933/howler/cronjobs/action_queue_worker.py

@@ -0,0 +1,82 @@
+"""Action queue worker cronjob — starts per-trigger action queue consumer threads.
+
+Auto-discovered by ``howler.cronjobs.setup_jobs`` when ``HWL_USE_JOB_SYSTEM``
+is enabled.  Instead of scheduling a periodic APScheduler job, this module
+starts a long-running daemon thread per trigger type that drains its
+dedicated action queue.
+"""
+
+import threading
+
+from apscheduler.schedulers.base import BaseScheduler
+
+from howler.common.logging import get_logger
+from howler.config import config
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.services.action_service import _get_action_queue, process_action_batch
+
+logger = get_logger(__file__)
+
+_threads: dict[str, threading.Thread] = {}
+
+BATCH_SIZE: int = config.system.action_queue.batch_size
+BATCH_TIMEOUT: int = config.system.action_queue.batch_timeout
+
+
+def run_worker(trigger: str) -> None:  # pragma: no cover – long-running loop, tested via process_action_batch
+    """Block on the action queue for *trigger* and process batches.
+
+    Accumulates up to ``BATCH_SIZE`` items or flushes after ``BATCH_TIMEOUT``
+    seconds, whichever comes first.
+    """
+    if not config.system.action_queue.enabled:
+        logger.info("Action queue worker disabled by configuration, not starting for trigger=%s", trigger)
+        return
+
+    queue = _get_action_queue(trigger)
+    logger.info(
+        "Action queue worker started for trigger=%s (batch_size=%d, timeout=%ds)",
+        trigger,
+        BATCH_SIZE,
+        BATCH_TIMEOUT,
+    )
+
+    batch: list[dict] = []
+
+    while True:
+        try:
+            item = queue.pop(blocking=True, timeout=BATCH_TIMEOUT)
+
+            if item is not None:
+                batch.append(item)
+
+            if len(batch) >= BATCH_SIZE or (item is None and batch):
+                logger.debug("Processing action batch of %d item(s) for trigger=%s", len(batch), trigger)
+                try:
+                    process_action_batch(trigger, batch)
+                    logger.info("Action batch complete: %d item(s) processed for trigger=%s", len(batch), trigger)
+                except Exception:
+                    logger.exception("Error processing action batch for trigger=%s", trigger)
+                finally:
+                    batch = []
+        except Exception:
+            logger.exception("Unexpected error in action queue worker loop for trigger=%s", trigger)
+
+
+def setup_job(sched: BaseScheduler):
+    """Start an action queue worker thread per trigger type if the action queue is enabled."""
+    global _threads
+
+    if not config.system.action_queue.enabled:
+        logger.info("Action queue worker disabled by configuration")
+        return
+
+    for trigger in VALID_TRIGGERS:
+        if trigger in _threads and _threads[trigger].is_alive():
+            logger.debug("Action queue worker thread for trigger=%s already running", trigger)
+            continue
+
+        thread = threading.Thread(target=run_worker, args=(trigger,), name=f"action-queue-{trigger}", daemon=True)
+        thread.start()
+        _threads[trigger] = thread
+        logger.info("Action queue worker thread started for trigger=%s", trigger)

{howler_api-3.4.0.dev929 → howler_api-3.4.0.dev933}/howler/odm/models/config.py

@@ -353,6 +353,19 @@ class ViewCleanup(BaseModel):
     )
 
 
+class ActionQueue(BaseModel):
+    """Action queue worker configuration.
+
+    Controls the background worker that buffers action execution requests
+    via a Redis queue, coalescing them into batches to reduce Elasticsearch
+    query load during ingestion spikes.
+    """
+
+    enabled: bool = Field(default=True, description="Enable the action queue worker?")
+    batch_size: int = Field(default=100, description="Max action items per batch.")
+    batch_timeout: int = Field(default=10, description="Seconds to wait before flushing a partial batch.")
+
+
 class System(BaseModel):
     """System-level configuration for Howler.
 
@@ -366,6 +379,8 @@ class System(BaseModel):
     "Retention Configuration"
     view_cleanup: ViewCleanup = ViewCleanup()
     "View Cleanup Configuration"
+    action_queue: ActionQueue = ActionQueue()
+    "Action Queue Worker Configuration"
 
 
 class UI(BaseModel):

{howler_api-3.4.0.dev929 → howler_api-3.4.0.dev933}/howler/services/action_service.py

@@ -9,13 +9,108 @@ from howler.common.exceptions import HowlerValueError
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.common.logging.audit import audit
+from howler.config import config
 from howler.odm.models.action import VALID_TRIGGERS, Action
 from howler.odm.models.user import User
+from howler.remote.datatypes.queues.named import NamedQueue
 from howler.utils.constants import TESTING
 from howler.utils.str_utils import sanitize_lucene_query
 
 logger = get_logger(__file__)
 
+# Per-trigger persistent queues for buffering action execution requests.
+_action_queues: dict[str, NamedQueue[dict]] = {}
+
+
+def _get_action_queue(trigger: str) -> NamedQueue[dict]:
+    """Return the action queue for *trigger*, creating it on first use.
+
+    Raises:
+        ValueError: If *trigger* is not in ``VALID_TRIGGERS``.
+    """
+    if trigger not in VALID_TRIGGERS:
+        raise HowlerValueError(f"Invalid trigger {trigger!r}. Must be one of {VALID_TRIGGERS}")
+
+    if trigger not in _action_queues:
+        _action_queues[trigger] = NamedQueue(
+            f"howler.action_queue.{trigger}",
+            host=config.core.redis.persistent.host,
+            port=config.core.redis.persistent.port,
+            private=False,
+        )
+
+    return _action_queues[trigger]
+
+
+def enqueue_action_execution(hit_ids: list[str], trigger: str = "create", user: Optional[User] = None) -> None:
+    """Buffer action execution by pushing to a Redis queue.
+
+    When the action queue is disabled in configuration, falls back to
+    calling ``bulk_execute_on_query`` directly for backwards compatibility.
+
+    Args:
+        hit_ids: List of hit IDs to execute actions against.
+        trigger: The trigger type (create, promote, demote, add_label, remove_label).
+        user: The user initiating the action.
+    """
+    if not hit_ids:
+        return
+
+    if trigger not in VALID_TRIGGERS:
+        raise HowlerValueError(f"Invalid trigger {trigger!r}. Must be one of {VALID_TRIGGERS}")
+
+    if not config.system.action_queue.enabled:
+        query = f"howler.id:({' OR '.join(sanitize_lucene_query(h) for h in hit_ids)})"
+        bulk_execute_on_query(query, trigger=trigger, user=user)
+        return
+
+    try:
+        _get_action_queue(trigger).push(
+            {
+                "hit_ids": hit_ids,
+                "user": user.as_primitives() if user is not None and hasattr(user, "as_primitives") else user,
+            }
+        )
+    except Exception:
+        logger.exception("Failed to enqueue action execution, falling back to direct execution")
+        query = f"howler.id:({' OR '.join(sanitize_lucene_query(h) for h in hit_ids)})"
+        bulk_execute_on_query(query, trigger=trigger, user=user)
+
+
+def process_action_batch(trigger: str, items: list[dict]) -> None:
+    """Process a batch of queued action execution requests for a single trigger.
+
+    Groups items by user and issues a single coalesced
+    ``bulk_execute_on_query`` call per group, reducing Elasticsearch load.
+
+    Args:
+        trigger: The trigger type this batch belongs to.
+        items: List of dicts with keys ``hit_ids`` and ``user``.
+    """
+    if not items:
+        return
+
+    groups: dict[str | None, tuple[list[str], Any]] = {}
+
+    for item in items:
+        user_data = item.get("user")
+        user_key = user_data["uname"] if isinstance(user_data, dict) and "uname" in user_data else None
+
+        if user_key not in groups:
+            groups[user_key] = ([], user_data)
+
+        groups[user_key][0].extend(item["hit_ids"])
+
+    for user_key, (hit_ids, user_data) in groups.items():
+        # Deduplicate IDs within a batch group
+        unique_ids = list(dict.fromkeys(hit_ids))
+        query = f"howler.id:({' OR '.join(sanitize_lucene_query(h) for h in unique_ids)})"
+
+        try:
+            bulk_execute_on_query(query, trigger=trigger, user=user_data)
+        except Exception:
+            logger.exception("Error processing action batch for trigger=%s user=%s", trigger, user_key)
+
 
 def validate_action(new_action: Any) -> Optional[Response]:  # noqa: C901
     """Validate a new action"""

{howler_api-3.4.0.dev929 → howler_api-3.4.0.dev933}/howler/services/hit_service.py

@@ -736,12 +736,12 @@ def transition_hit(
         # Commit database changes before executing bulk actions
         datastore().hit.commit()
 
-        # Build query for all processed hits (primary + children)
-        all_processed_hits = [primary_hit] + child_hits
-        hit_query = f"howler.id:({' OR '.join(h['howler']['id'] for h in all_processed_hits)})"
+        # Build query for all processed hits (primary + children), excluding missing hits
+        all_processed_hits = [primary_hit] + [ch for ch in child_hits if ch]
+        hit_ids = [h["howler"]["id"] for h in all_processed_hits]
 
-        # Execute bulk actions on all hits
-        action_service.bulk_execute_on_query(hit_query, trigger=trigger, user=user)
+        # Enqueue action execution for all hits
+        action_service.enqueue_action_execution(hit_ids, trigger=trigger, user=user)
 
         # Emit events for all processed hits to notify other systems
         for processed_hit in all_processed_hits:

{howler_api-3.4.0.dev929 → howler_api-3.4.0.dev933}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "3.4.0.dev929"
+version = "3.4.0.dev933"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",