howler-api 4.1.0.dev1346__tar.gz → 4.1.0.dev1368__tar.gz

{howler_api-4.1.0.dev1346 → howler_api-4.1.0.dev1368}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.1.0.dev1346
+Version: 4.1.0.dev1368
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -50,7 +50,7 @@ Requires-Dist: python-datemath (==3.0.3)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
 Requires-Dist: pytz (>=2025.2,<2026.0)
 Requires-Dist: pyyaml (==6.0.3)
-Requires-Dist: redis (==8.0.1)
+Requires-Dist: redis (==8.1.0)
 Requires-Dist: requests (==2.34.2)
 Requires-Dist: tzdata (>=2026.1,<2027.0)
 Requires-Dist: validators (>=0.34,<0.36)

{howler_api-4.1.0.dev1346 → howler_api-4.1.0.dev1368}/howler/api/v1/hit.py

@@ -30,7 +30,7 @@ from howler.odm.models.hit import Hit
 from howler.odm.models.howler_data import Comment, HitOperationType, HitStatusTransition
 from howler.odm.models.user import User
 from howler.security import api_login
-from howler.services import action_service, analytic_service, comms_service, hit_service
+from howler.services import action_service, analytic_service, comms_service, correlation_service, hit_service
 from howler.utils.constants import DEBUG_FORCE_REFRESH
 from howler.utils.str_utils import sanitize_lucene_query
 
@@ -130,7 +130,9 @@ def create_hits(user: User, **kwargs):
 
             hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
             analytic_service.save_from_hits(odms, user, refresh=refresh)
-            action_service.enqueue_action_execution([odm.howler.id for odm in odms], trigger="create", user=user)
+            ids = [odm.howler.id for odm in odms]
+            action_service.enqueue_action_execution(ids, trigger="create", user=user)
+            correlation_service.enqueue_for_correlation(ids)
 
         response_body["warnings"] = warnings
 

{howler_api-4.1.0.dev1346 → howler_api-4.1.0.dev1368}/howler/api/v1/tool.py

@@ -13,7 +13,7 @@ from howler.odm.base import _Field
 from howler.odm.models.hit import Hit
 from howler.odm.models.user import User
 from howler.security import api_login
-from howler.services import action_service, analytic_service, hit_service
+from howler.services import action_service, analytic_service, correlation_service, hit_service
 from howler.utils.constants import DEBUG_FORCE_REFRESH
 from howler.utils.dict_utils import flatten
 from howler.utils.isotime import now_as_iso
@@ -233,6 +233,8 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
         hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
         analytic_service.save_from_hits(odms, user, refresh=refresh)
 
-    action_service.enqueue_action_execution([entry["id"] for entry in out], trigger="create", user=user)
+    ids = [entry["id"] for entry in out]
+    action_service.enqueue_action_execution(ids, trigger="create", user=user)
+    correlation_service.enqueue_for_correlation(ids)
 
     return created(out, warnings=warnings)

{howler_api-4.1.0.dev1346 → howler_api-4.1.0.dev1368}/howler/api/v2/ingest.py

@@ -11,7 +11,6 @@ from howler.common.exceptions import HowlerException, HowlerValueError
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.common.swagger import generate_swagger_docs
-from howler.config import CORRELATION_QUEUE_NAME, config
 from howler.datastore.collection import ESCollection
 from howler.datastore.exceptions import DataStoreException
 from howler.datastore.howler_store import INDEXES
@@ -19,9 +18,8 @@ from howler.datastore.operations import OdmHelper, OdmUpdateOperation
 from howler.odm.models.event import Event
 from howler.odm.models.hit import Hit
 from howler.odm.models.user import User
-from howler.remote.datatypes.queues.named import NamedQueue
 from howler.security import api_login
-from howler.services import event_service, hit_service
+from howler.services import correlation_service, event_service, hit_service
 from howler.utils.dict_utils import flatten
 
 MAX_COMMENT_LEN = 5000
@@ -36,24 +34,6 @@ logger = get_logger(__file__)
 
 hit_helper = OdmHelper(Hit)
 
-# Persistent queue for the correlation worker to consume newly ingested hit IDs.
-_ingestion_queue: NamedQueue[str] | None = None
-
-
-def _get_ingestion_queue() -> NamedQueue[str]:
-    """Return the shared ingestion queue, creating it on first use."""
-    global _ingestion_queue
-
-    if _ingestion_queue is None:
-        _ingestion_queue = NamedQueue(
-            CORRELATION_QUEUE_NAME,
-            host=config.core.redis.persistent.host,
-            port=config.core.redis.persistent.port,
-            private=False,
-        )
-
-    return _ingestion_queue
-
 
 @generate_swagger_docs()
 @ingest_api.route("/<index>", methods=["POST"])
@@ -120,10 +100,7 @@ def create(index: str, user: User, **kwargs):
 
     # Enqueue newly created hit IDs for the correlation worker.
     if ids:
-        try:
-            _get_ingestion_queue().push(*ids)
-        except Exception:
-            logger.exception("Failed to enqueue hit IDs for correlation")
+        correlation_service.enqueue_for_correlation(ids)
 
     return created(ids, warnings=warnings)
 

{howler_api-4.1.0.dev1346 → howler_api-4.1.0.dev1368}/howler/services/correlation_service.py

@@ -37,6 +37,40 @@ def _normalize_utc(ts: datetime) -> datetime:
     return ts
 
 
+# Persistent queue for the correlation worker to consume newly ingested hit IDs.
+_ingestion_queue: NamedQueue[str] | None = None
+
+
+def _get_ingestion_queue() -> NamedQueue[str]:
+    """Return the shared ingestion queue, creating it on first use."""
+    global _ingestion_queue
+
+    if _ingestion_queue is None:
+        _ingestion_queue = NamedQueue(
+            CORRELATION_QUEUE_NAME,
+            host=config.core.redis.persistent.host,
+            port=config.core.redis.persistent.port,
+            private=False,
+        )
+
+    return _ingestion_queue
+
+
+def enqueue_for_correlation(ids: list[str]) -> None:
+    """Enqueue record IDs for correlation processing.
+
+    Args:
+        ids: List of record IDs to enqueue for correlation.
+
+    Raises:
+        HowlerRuntimeError: If enqueueing fails.
+    """
+    try:
+        _get_ingestion_queue().push(*ids)
+    except Exception:
+        logger.exception("Error on queuing for correlation")
+
+
 def get_active_rules() -> list[tuple[str, CaseRule]]:  # noqa: C901
     """Return all active (enabled, non-expired) rules across every case.
 
@@ -179,23 +213,13 @@ def process_batch(record_ids: list[str]) -> int:  # noqa: C901
     return added
 
 
-def _build_queue() -> NamedQueue[str]:
-    """Create the NamedQueue backed by persistent Redis."""
-    return NamedQueue(
-        CORRELATION_QUEUE_NAME,
-        host=config.core.redis.persistent.host,
-        port=config.core.redis.persistent.port,
-        private=False,
-    )
-
-
 def run_worker() -> None:  # pragma: no cover – long-running loop, tested via process_batch
     """Block on the ingestion queue and process batches of record IDs.
 
     Accumulates up to ``BATCH_SIZE`` IDs or flushes after ``BATCH_TIMEOUT``
     seconds, whichever comes first.
     """
-    queue = _build_queue()
+    queue = _get_ingestion_queue()
     logger.info("Correlation worker started (batch_size=%d, timeout=%ds)", BATCH_SIZE, BATCH_TIMEOUT)
 
     batch: list[str] = []

{howler_api-4.1.0.dev1346 → howler_api-4.1.0.dev1368}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.1.0.dev1346"
+version = "4.1.0.dev1368"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",
@@ -211,7 +211,7 @@ pyjwt = "2.13.0"
 python-baseconv = "1.2.2"
 python-datemath = "3.0.3"
 pyyaml = "6.0.3"
-redis = "8.0.1"
+redis = "8.1.0"
 requests = "2.34.2"
 wsproto = "1.3.2"
 chevron = "0.14.0"