howler-api 4.0.0.dev799__tar.gz → 4.0.0.dev841__tar.gz

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev799
+Version: 4.0.0.dev841
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/api/socket.py

@@ -7,11 +7,11 @@ from flask import Blueprint, request
 from opentelemetry import trace
 
 import howler.services.event_service as event_service
+import howler.services.viewer_service as viewer_service
 from howler.api import ok, unauthorized
 from howler.common.logging import get_logger
-from howler.datastore.operations import OdmHelper
 from howler.helper.ws import ConnectionClosed, Server
-from howler.odm.models.hit import Hit
+from howler.security import api_login
 from howler.security.socket import websocket_auth, ws_response
 from howler.utils.socket_utils import check_action
 
@@ -24,13 +24,16 @@ socket_api._doc = "Endpoints concerning websocket connectivity between the clien
 logger = get_logger(__file__)
 tracer = trace.get_tracer(__name__)
 
-hit_helper = OdmHelper(Hit)
-
 
 @tracer.start_as_current_span(f"{__name__}.emit")
 @socket_api.route("/emit/<event>", methods=["POST"])
 def emit(event: str):
-    """Emit an event to all listening websockets"""
+    """Emit an event to all listening websockets.
+
+    .. deprecated::
+        This endpoint is deprecated. Events are now propagated via Redis pubsub
+        and no longer require a dedicated websocket pod.
+    """
     if "Authorization" not in request.headers:
         return unauthorized(err="Missing authorization header")
 
@@ -49,10 +52,25 @@ def emit(event: str):
     return ok()
 
 
+@tracer.start_as_current_span(f"{__name__}.get_viewers")
+@socket_api.route("/viewers/<entity_id>", methods=["GET"])
+@api_login(audit=False, required_priv=["R"])
+def get_viewers(entity_id: str, **kwargs):
+    """Get the list of users currently viewing the specified entity
+
+    Variables:
+    entity_id       => The ID of the entity to get viewers for
+
+    Result Example:
+    ["user1", "user2"]
+    """
+    return ok(viewer_service.get_viewers(entity_id))
+
+
 @tracer.start_as_current_span(f"{__name__}.connect")
 @socket_api.route("/connect", websocket=True)  # type: ignore
 @websocket_auth(required_priv=["R"])
-def connect(ws: Server, *args: Any, ws_id: str, **kwargs):
+def connect(ws: Server, *args: Any, ws_id: str, **kwargs):  # noqa: C901
     """Connect to the server to monitor for updates via websocket
 
     Variables:
@@ -78,10 +96,20 @@ def connect(ws: Server, *args: Any, ws_id: str, **kwargs):
         logger.debug("Sending action: %s", data)
         ws.send(ws_response("action", data))
 
+    def send_case(data: dict[str, Any]):
+        logger.debug("Sending case update: %s", data.get("case", {}).get("case_id", "unknown"))
+        ws.send(ws_response("cases", data))
+
+    def send_viewers_update(data: dict[str, Any]):
+        logger.debug("Sending viewers update: %s", data.get("id", "unknown"))
+        ws.send(ws_response("viewers_update", data))
+
     try:
         event_service.on("hits", send_hit)
         event_service.on("broadcast", send_broadcast)
         event_service.on("action", send_action)
+        event_service.on("cases", send_case)
+        event_service.on("viewers_update", send_viewers_update)
         while ws.connected:
             data = ws.receive(10)
             if data:
@@ -113,6 +141,8 @@ def connect(ws: Server, *args: Any, ws_id: str, **kwargs):
         event_service.off("hits", send_hit)
         event_service.off("broadcast", send_broadcast)
         event_service.off("action", send_action)
+        event_service.off("cases", send_case)
+        event_service.off("viewers_update", send_viewers_update)
 
         for id, action, broadcast in outstanding_actions:
             outstanding_actions = check_action(id, action, broadcast, outstanding_actions=outstanding_actions, **kwargs)

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/api/v2/case.py

@@ -290,7 +290,7 @@ def delete_item(case_id: str, **kwargs):
 
 
 @generate_swagger_docs()
-@case_api.route("/<case_id>/items", methods=["PATCH"])
+@case_api.route("/<case_id>/items", methods=["PUT"])
 @api_login(required_priv=["R", "W"])
 def rename_item(case_id: str, **kwargs):
     """Rename (re-path) an item within a case
@@ -331,3 +331,105 @@ def rename_item(case_id: str, **kwargs):
         return internal_error(err=str(e))
     except (InvalidDataException, NotFoundException) as e:
         return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@case_api.route("/<id>/rules", methods=["POST"])
+@api_login(required_priv=["R", "W"])
+def add_rule(id: str, user: User, **kwargs):
+    """Add a correlation rule to a case
+
+    Creates a new correlation rule that will match incoming alerts into the case.
+    The rule's id and author are generated server-side.
+
+    Variables:
+    id       => The id of the case to add a rule to
+
+    Arguments:
+    None
+
+    Data Block:
+    {
+        "query": "howler.analytic:Suspicious*",
+        "destination": "alerts/{{howler.analytic}}",
+        "timeframe": "2026-05-06T00:00:00Z"   // optional, null means no expiry
+    }
+
+    Result Example:
+    {
+        ...case     # The updated case data
+    }
+    """
+    body = request.json
+
+    if not body or not isinstance(body, dict):
+        return bad_request(err="Request body must be a JSON object with rule data.")
+
+    try:
+        return ok(case_service.add_case_rule(id, body, user))
+    except NotFoundException as e:
+        return not_found(err=str(e))
+    except InvalidDataException as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@case_api.route("/<id>/rules/<rule_id>", methods=["DELETE"])
+@api_login(required_priv=["R", "W"])
+def delete_rule(id: str, rule_id: str, user: User, **kwargs):
+    """Delete a correlation rule from a case
+
+    Variables:
+    id        => The id of the case
+    rule_id   => The id of the rule to delete
+
+    Arguments:
+    None
+
+    Result Example:
+    {
+        ...case     # The updated case data
+    }
+    """
+    try:
+        return ok(case_service.remove_case_rule(id, rule_id, user))
+    except NotFoundException as e:
+        return not_found(err=str(e))
+
+
+@generate_swagger_docs()
+@case_api.route("/<id>/rules/<rule_id>", methods=["PUT"])
+@api_login(required_priv=["R", "W"])
+def update_rule(id: str, rule_id: str, user: User, **kwargs):
+    """Update a correlation rule on a case
+
+    Allows updating individual fields on a rule: enabled, query, destination, timeframe.
+
+    Variables:
+    id        => The id of the case
+    rule_id   => The id of the rule to update
+
+    Arguments:
+    None
+
+    Data Block:
+    {
+        "enabled": false
+    }
+
+    Result Example:
+    {
+        ...case     # The updated case data
+    }
+    """
+    body = request.json
+
+    if not body or not isinstance(body, dict):
+        return bad_request(err="Request body must be a JSON object with fields to update.")
+
+    try:
+        return ok(case_service.update_case_rule(id, rule_id, body, user))
+    except NotFoundException as e:
+        return not_found(err=str(e))
+    except InvalidDataException as e:
+        return bad_request(err=str(e))

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/api/v2/ingest.py

@@ -9,6 +9,7 @@ from howler.common.exceptions import HowlerException, HowlerValueError
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.common.swagger import generate_swagger_docs
+from howler.config import config
 from howler.datastore.collection import ESCollection
 from howler.datastore.exceptions import DataStoreException
 from howler.datastore.howler_store import INDEXES
@@ -16,6 +17,7 @@ from howler.datastore.operations import OdmHelper, OdmUpdateOperation
 from howler.odm.models.hit import Hit
 from howler.odm.models.observable import Observable
 from howler.odm.models.user import User
+from howler.remote.datatypes.queues.named import NamedQueue
 from howler.security import api_login
 from howler.services import hit_service, observable_service
 from howler.utils.dict_utils import flatten
@@ -32,6 +34,24 @@ logger = get_logger(__file__)
 
 hit_helper = OdmHelper(Hit)
 
+# Persistent queue for the correlation worker to consume newly ingested hit IDs.
+_ingestion_queue: NamedQueue[str] | None = None
+
+
+def _get_ingestion_queue() -> NamedQueue[str]:
+    """Return the shared ingestion queue, creating it on first use."""
+    global _ingestion_queue
+
+    if _ingestion_queue is None:
+        _ingestion_queue = NamedQueue(
+            "howler.ingestion_queue",
+            host=config.core.redis.persistent.host,
+            port=config.core.redis.persistent.port,
+            private=False,
+        )
+
+    return _ingestion_queue
+
 
 @generate_swagger_docs()
 @ingest_api.route("/<index>", methods=["POST"])
@@ -93,6 +113,13 @@ def create(index: str, user: User, **kwargs):
             logger.exception("Ingestion failed.")
             return bad_request(err=f"Ingestion failure on record at index {i}: {e}")
 
+    # Enqueue newly created hit IDs for the correlation worker.
+    if ids:
+        try:
+            _get_ingestion_queue().push(*ids)
+        except Exception:
+            logger.exception("Failed to enqueue hit IDs for correlation")
+
     return created(ids, warnings=warnings)
 
 
@@ -120,15 +147,20 @@ def delete(indexes: str, user: User, **kwargs):
      "success": True             # Deleting the hits succeded
     }
     """
-    hit_ids = request.json
+    ids = request.json
 
-    if hit_ids is None:
+    if ids is None:
         return bad_request(err="No hit ids were sent.")
 
     if "admin" not in user["type"]:
         return forbidden(err="Cannot delete hit, only administrators are permitted to delete.")
 
-    index_list = indexes.split(",")  # noqa: F841
+    index_list = indexes.split(",")
+
+    ds = datastore()
+
+    if non_existing_hit_ids := [id for id in ids if all(not ds[index].exists(id) for index in index_list)]:
+        return not_found(err=f"Record ids [{','.join(non_existing_hit_ids)}] do not exist.")
 
     # TODO: Reimplement in a generic function
     # hit_service.delete_hits(hit_ids, indexes=index_list)

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/app.py

@@ -174,6 +174,11 @@ else:
 if HWL_USE_WEBSOCKET_API or DEBUG:
     logger.debug("Enabled Websocket API")
     app.register_blueprint(socket_api)
+
+    # Start the Redis pubsub watcher so this pod receives events from all pods
+    import howler.services.event_service as event_service
+
+    event_service.start_watcher()
 else:
     logger.info("Disabled Websocket API")
 

howler_api-4.0.0.dev841/howler/cronjobs/correlation.py

@@ -0,0 +1,36 @@
+"""Correlation cronjob — starts the correlation worker thread.
+
+Auto-discovered by ``howler.cronjobs.setup_jobs`` when ``HWL_USE_JOB_SYSTEM``
+is enabled.  Instead of scheduling a periodic APScheduler job, this module
+starts a long-running daemon thread that drains the ingestion queue.
+"""
+
+import threading
+
+from apscheduler.schedulers.base import BaseScheduler
+
+from howler.common.logging import get_logger
+from howler.odm.models.config import config
+
+logger = get_logger(__file__)
+
+_thread: threading.Thread | None = None
+
+
+def setup_job(sched: BaseScheduler):
+    """Start the correlation worker thread if correlation is enabled."""
+    global _thread
+
+    if not config.system.correlation.enabled:
+        logger.info("Correlation worker disabled by configuration")
+        return
+
+    if _thread is not None and _thread.is_alive():
+        logger.debug("Correlation worker thread already running")
+        return
+
+    from howler.services.correlation_service import run_worker
+
+    _thread = threading.Thread(target=run_worker, name="correlation-worker", daemon=True)
+    _thread.start()
+    logger.info("Correlation worker thread started")

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/datastore/collection.py

@@ -1365,6 +1365,7 @@ class ESCollection(Generic[ModelType]):
                     extra_fields["_index"] = result["_index"]
                 if "*" in fields:
                     fields = None
+
                 return self.model_class(source_data, mask=fields, docid=item_id, extra_fields=extra_fields)
             else:
                 source_data = recursive_update(source_data, extra_fields, allow_recursion=False)
@@ -1692,6 +1693,32 @@ class ESCollection(Generic[ModelType]):
 
         return ret_data
 
+    @overload
+    def stream_search(
+        self,
+        query: str,
+        fl: str | None = None,
+        filters: list[str] | str | None = None,
+        access_control: typing.Any = None,
+        item_buffer_size: int = 200,
+        *,
+        as_obj: Literal[True] = True,
+        use_archive: bool = False,
+    ) -> typing.Generator[ModelType, None, None]: ...
+
+    @overload
+    def stream_search(
+        self,
+        query: str,
+        fl: str | None = None,
+        filters: list[str] | str | None = None,
+        access_control: typing.Any = None,
+        item_buffer_size: int = 200,
+        *,
+        as_obj: Literal[False],
+        use_archive: bool = False,
+    ) -> typing.Generator[dict[str, typing.Any], None, None]: ...
+
     def stream_search(
         self,
         query,

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/odm/helper.py

@@ -249,8 +249,6 @@ def generate_useful_hit(  # noqa: C901
     except IndexError:
         pass
 
-    hit.howler.viewers = []
-
     hit.howler.dossier = [
         Lead(
             {
@@ -446,8 +444,6 @@ def generate_useful_observable(  # noqa: C901
         ),
     ]
 
-    observable.howler.viewers = []
-
     return observable
 
 

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/odm/models/case.py

@@ -8,6 +8,19 @@ from howler.utils.compat import StrEnum
 
 CASE_ITEM_TYPES = {"observable", "hit", "case", "lead", "reference"}
 
+RULE_INDEX_TYPES = {"hit", "observable"}
+
+
+class RuleIndexTypes(StrEnum):
+    """Enumeration of valid index types for case rules.
+
+    Determines which Elasticsearch indexes a case rule query runs against
+    during correlation.
+    """
+
+    HIT = "hit"
+    OBSERVABLE = "observable"
+
 
 class CaseItemTypes(StrEnum):
     """Enumeration of valid case item types.
@@ -69,8 +82,20 @@ class CaseItem(odm.Model):
 
 @odm.model(index=True, store=True, description="Rule used to place/query data into case paths.")
 class CaseRule(odm.Model):
+    rule_id: str = odm.UUID(description="Unique rule identifier.")
     destination: str = odm.Keyword(description="Destination case path template.")
     query: str = odm.Keyword(description="Lucene query used by this rule.")
+    author: str = odm.Keyword(description="Username who created the rule.")
+    enabled: bool = odm.Boolean(default=True, description="Whether the rule is currently active.")
+    timeframe: Optional[str] = odm.Optional(
+        odm.Date(description="ISO datetime when rule expires. Null means no expiry."),
+        default=None,
+    )
+    indexes: list[str] = odm.List(
+        odm.Enum(values=RuleIndexTypes),
+        default=[RuleIndexTypes.HIT],
+        description="Indexes to run this rule against (hit, observable, or both).",
+    )
 
 
 @odm.model(index=True, store=True, description="Task associated with a case item path.")

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/odm/models/config.py

@@ -353,6 +353,18 @@ class ViewCleanup(BaseModel):
     )
 
 
+class Correlation(BaseModel):
+    """Correlation worker configuration.
+
+    Controls the background worker that matches newly ingested alerts
+    against active case rules.
+    """
+
+    enabled: bool = Field(default=True, description="Enable the correlation worker?")
+    batch_size: int = Field(default=100, description="Max alerts per batch.")
+    batch_timeout: int = Field(default=10, description="Seconds to wait before flushing a partial batch.")
+
+
 class System(BaseModel):
     """System-level configuration for Howler.
 
@@ -366,6 +378,8 @@ class System(BaseModel):
     "Retention Configuration"
     view_cleanup: ViewCleanup = ViewCleanup()
     "View Cleanup Configuration"
+    correlation: Correlation = Correlation()
+    "Correlation Worker Configuration"
 
 
 class UI(BaseModel):

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/odm/models/howler_data.py

@@ -293,7 +293,3 @@ class HowlerData(odm.Model):
     dossier: list[Lead] = odm.List(
         odm.Compound(Lead), default=[], description="A list of leads forming the dossier associated with this hit"
     )
-    viewers: list[str] = odm.List(
-        odm.Keyword(description="A list of users currently viewing the hit"),
-        default=[],
-    )

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/odm/models/observable.py

@@ -101,10 +101,6 @@ class ObservableData(odm.Model):
         default=[],
         description="A list of changes to the observable with timestamps and attribution.",
     )
-    viewers: list[str] = odm.List(
-        odm.Keyword(description="A list of users currently viewing the observable"),
-        default=[],
-    )
 
 
 @odm.model(

{howler_api-4.0.0.dev799 → howler_api-4.0.0.dev841}/howler/odm/random_data.py

@@ -19,7 +19,7 @@ import importlib
 import json
 import random
 import textwrap
-from datetime import datetime
+from datetime import datetime, timedelta
 from random import choice, randint, sample
 from typing import Any, Callable, cast
 from uuid import uuid4
@@ -866,9 +866,35 @@ def create_cases(ds: HowlerDatastore, num_cases: int = 5):
                 "enrichments": [],
                 "rules": [
                     {
-                        "destination": "alerts/{{howler.id}}",
-                        "query": f"destination.domain:{choice(threat_pool)}",
+                        "destination": choice(
+                            [
+                                "alerts/{{howler.analytic}}",
+                                "incoming/{{event.kind}}",
+                                "alerts/{{howler.analytic}}/{{event.category}}",
+                                "correlated/{{source.ip}}",
+                                "triage/{{howler.escalation}}",
+                            ]
+                        ),
+                        "query": choice(
+                            [
+                                f"destination.domain:{choice(threat_pool)}",
+                                "source.ip:10.0.0.0/8 AND howler.analytic:Suspicious*",
+                                "event.category:authentication AND event.outcome:failure",
+                                "howler.escalation:focus OR howler.escalation:crisis",
+                                f"destination.domain:{choice(threat_pool)} AND event.kind:alert",
+                            ]
+                        ),
+                        "author": choice(selected_participants or ["admin"]),
+                        "enabled": choice([True, True, True, False]),
+                        "timeframe": choice(
+                            [
+                                (datetime.now() + timedelta(days=randint(7, 28))).isoformat(),
+                                (datetime.now() + timedelta(days=randint(7, 28))).isoformat(),
+                                None,
+                            ]
+                        ),
                     }
+                    for _ in range(randint(1, 3))
                 ],
                 "tasks": tasks,
             }