howler-api 4.1.0.dev1368__tar.gz → 4.1.0.dev1387__tar.gz

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.1.0.dev1368
+Version: 4.1.0.dev1387
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/howler/actions/add_to_case.py

@@ -13,8 +13,7 @@ OPERATION_ID = "add_to_case"
 def execute(  # noqa: C901
     query: str,
     case_id: Optional[str] = None,
-    path: str = "related",
-    title_template: str = "{{howler.analytic}} ({{howler.id}})",
+    destination: str = "related/{{howler.analytic}} ({{howler.id}})",
     **kwargs,
 ):
     """Add matching alerts to a given case.
@@ -22,10 +21,9 @@ def execute(  # noqa: C901
     Args:
         query (str): The query on which to apply this automation.
         case_id (str): The ID of the case to add the alerts to.
-        path (str): The path within the case at which to place the alerts. Defaults to "related".
-        title_template (str): A Mustache-compatible template string used to generate each item's
-            name. The hit's fields are available as template variables.
-            Defaults to "{{howler.analytic}} ({{howler.id}})".
+        destination (str): A Mustache-compatible template string for the case path at which each
+            alert will be placed, in the form "path/to/parent/name". The hit's fields are
+            available as template variables. Defaults to "related/{{howler.analytic}} ({{howler.id}})".
     """
     if not case_id:
         return [
@@ -67,15 +65,15 @@ def execute(  # noqa: C901
     added = []
 
     for hit in hits:
-        title = chevron.render(title_template, hit.as_primitives())
-        item_path = f"{path.rstrip('/')}/{title}" if path else title
+        rendered_destination = chevron.render(destination, hit.as_primitives())
         try:
-            path, name = item_path.rsplit("/", maxsplit=1)
+            item_path, name = rendered_destination.rsplit("/", maxsplit=1)
         except ValueError:
-            name = item_path
+            item_path = None
+            name = rendered_destination
 
         try:
-            parent = case_service.get_parent_from_path(case, path, create_if_missing=True)
+            parent = case_service.get_parent_from_path(case, item_path, create_if_missing=True)
 
             case_service.append_case_item(
                 case,
@@ -131,8 +129,7 @@ def specification():
             {
                 "args": {
                     "case_id": [],
-                    "path": [],
-                    "title_template": [],
+                    "destination": [],
                 },
                 "options": {},
             }

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/howler/api/v2/ingest.py

@@ -39,7 +39,7 @@ hit_helper = OdmHelper(Hit)
 @ingest_api.route("/<index>", methods=["POST"])
 @api_login(required_priv=["W"])
 @parse_parameters(refresh=parse_refresh)
-def create(index: str, user: User, **kwargs):
+def create(index: str, user: User, *, refresh: Literal["true", "false", "wait_for"] | None = None, **kwargs):
     """Create new records in a given index.
 
     Variables:
@@ -75,11 +75,9 @@ def create(index: str, user: User, **kwargs):
     if not isinstance(records, list):
         return bad_request(err="JSON Payload must be a list of records.")
     ignore_extra_values = request.args.get("ignore_extra_values", False, type=lambda v: v.lower() == "true")
-    refresh = cast(Literal["true", "false", "wait_for"] | None, kwargs.get("refresh"))
 
-    ids: list[str] = []
+    odms: list = []
     warnings = []
-    refresh_kwargs: dict[str, Literal["true", "false", "wait_for"] | None] = {"refresh": refresh}
     for i, record in enumerate(records):
         try:
             odm: Hit | Event
@@ -87,18 +85,22 @@ def create(index: str, user: User, **kwargs):
                 odm, _warnings = event_service.convert_event(
                     record, unique=True, ignore_extra_values=ignore_extra_values
                 )
-                event_service.create_event(odm.howler.id, odm, user.uname, skip_exists=True, **refresh_kwargs)
             else:
                 odm, _warnings = hit_service.convert_hit(record, unique=True, ignore_extra_values=ignore_extra_values)
-                hit_service.create_hit(odm.howler.id, odm, user.uname, skip_exists=True, refresh=refresh)
 
-            ids.append(odm.howler.id)
+            odms.append(odm)
             warnings.extend(_warnings)
         except HowlerException as e:
             logger.exception("Ingestion failed.")
             return bad_request(err=f"Ingestion failure on record at index {i}: {e}")
 
+    if index == "event":
+        event_service.create_events(odms, user.uname, overwrite=False, refresh=refresh)
+    else:
+        hit_service.create_hits(odms, user.uname, overwrite=False, refresh=refresh)
+
     # Enqueue newly created hit IDs for the correlation worker.
+    ids = [odm.howler.id for odm in odms]
     if ids:
         correlation_service.enqueue_for_correlation(ids)
 

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/howler/app.py

@@ -71,6 +71,22 @@ from howler.telemetry import setup_telemetry
 logger = get_logger(__file__)
 
 
+def _should_start_background_services() -> bool:
+    """Return whether this process should start long-lived background services.
+
+    Flask's development reloader imports this module in a parent process and
+    again in its serving child. The functional test wrapper sets
+    ``HWL_START_BACKGROUND_SERVICES`` only for its no-reload Flask server,
+    preventing pytest imports from starting competing workers.
+    """
+    return (
+        not DEBUG
+        or os.environ.get("WERKZEUG_RUN_MAIN") == "true"
+        or os.environ.get("HWL_START_BACKGROUND_SERVICES", "").lower() == "true"
+        or Path(sys.argv[0]).name.lower().startswith("gunicorn")
+    )
+
+
 app = Flask(
     "howler-api",
     static_url_path="/api/static",
@@ -177,14 +193,15 @@ if HWL_USE_WEBSOCKET_API or DEBUG:
     logger.debug("Enabled Websocket API")
     app.register_blueprint(socket_api)
 
-    # Start the Redis pubsub watcher so this pod receives events from all pods
-    import howler.services.comms_service as comms_service
+    if _should_start_background_services():
+        # Start the Redis pubsub watcher so this pod receives events from all pods
+        import howler.services.comms_service as comms_service
 
-    comms_service.start_watcher()
+        comms_service.start_watcher()
 else:
     logger.info("Disabled Websocket API")
 
-if HWL_USE_JOB_SYSTEM or DEBUG:
+if HWL_USE_JOB_SYSTEM and _should_start_background_services():
     setup_jobs()
 
 

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/howler/config.py

@@ -21,7 +21,7 @@ MAX_CLASSIFICATION: str = CLASSIFICATION.UNRESTRICTED
 HWL_UNSECURED_UI = os.environ.get("HWL_UNSECURED_UI", "false").lower() == "true"
 HWL_USE_REST_API = os.environ.get("HWL_USE_REST_API", "true").lower() == "true"
 HWL_USE_WEBSOCKET_API = os.environ.get("HWL_USE_WEBSOCKET_API", "false").lower() == "true"
-HWL_USE_JOB_SYSTEM = os.environ.get("HWL_USE_JOB_SYSTEM", "false").lower() == "true"
+HWL_USE_JOB_SYSTEM = os.environ.get("HWL_USE_JOB_SYSTEM", "false").lower() == "true" or DEBUG
 HWL_ENABLE_COVERAGE = os.environ.get("HWL_ENABLE_COVERAGE", "false").lower() == "true"
 HWL_PLUGIN_DIRECTORY = os.environ.get("HWL_PLUGIN_DIRECTORY", "/etc/howler/plugins")
 CORRELATION_QUEUE_NAME = os.environ.get("HWL_CORRELATION_QUEUE_NAME", "howler.ingestion_queue")

howler_api-4.1.0.dev1387/howler/datastore/bulk.py

@@ -0,0 +1,272 @@
+"""Build newline-delimited JSON payloads for Elasticsearch bulk requests.
+
+``ElasticBulkPlan`` accumulates Elasticsearch bulk operations and renders them
+as NDJSON suitable for the ``_bulk`` endpoint. Documents can be serialized
+through an ODM model, and update operations can be limited to selected fields
+to avoid overwriting unrelated stored data.
+"""
+
+import json
+import warnings
+from copy import deepcopy
+from typing import List, Optional
+
+from howler import odm
+from howler.config import config
+from howler.datastore.utils import expand_field_patterns, prune_to_paths
+
+_OPERATION_GROUP = tuple[str] | tuple[str, str]
+
+ELASTIC_MAX_REQUEST_SIZE = config.datastore.max_request_size
+DEFAULT_BATCH_SIZE = config.datastore.request_batch_size
+
+
+class ElasticBulkPlan(object):
+    """Accumulate and render Elasticsearch bulk API operations.
+
+    Unless an operation supplies an explicit ``index``, insert, index, and
+    upsert operations target the first configured index. Delete and update
+    operations target every configured index, which supports collections that
+    span multiple physical indices.
+
+    Args:
+        indexes: Elasticsearch indices available to the plan. The first index
+            is the default target for single-index operations.
+        model: Optional ODM model used to serialize document values before
+            they are added to the request.
+    """
+
+    def __init__(self, indexes: List[str], model: Optional[type[odm.Model]] = None):
+        self.indexes = indexes
+        self.model = model
+        self.operations: list[_OPERATION_GROUP] = []
+
+    @property
+    def empty(self):
+        """Whether the plan has no queued operations.
+
+        Returns:
+            ``True`` when no bulk operation has been added; otherwise,
+            ``False``.
+        """
+        return len(self.operations) == 0
+
+    def add_delete_operation(self, doc_id, index=None):
+        """Queue a document delete operation.
+
+        Args:
+            doc_id: Identifier of the document to delete.
+            index: Explicit index to target. When omitted, queues one delete
+                operation for each configured index.
+        """
+        if index:
+            self.operations.append((json.dumps({"delete": {"_index": index, "_id": doc_id}}),))
+        else:
+            for cur_index in self.indexes:
+                self.operations.append((json.dumps({"delete": {"_index": cur_index, "_id": doc_id}}),))
+
+    def add_insert_operation(self, doc_id: str, doc, index=None):
+        """Queue a create-only document insert operation.
+
+        The Elasticsearch ``create`` action fails if a document with ``doc_id``
+        already exists. The document's ``id`` field is set to ``doc_id`` in the
+        serialized request body.
+
+        Args:
+            doc_id: Identifier to assign to the new document.
+            doc: Document data or an instance of the configured ODM model.
+            index: Explicit index to target. When omitted, uses the first
+                configured index.
+        """
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"create": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps(saved_doc),
+            )
+        )
+
+    def add_index_operation(self, doc_id, doc, index=None):
+        """Queue a document index operation.
+
+        The Elasticsearch ``index`` action creates the document or replaces an
+        existing document with the same identifier. The document's id field is
+        set to doc_id in the serialized request body.
+
+        Args:
+            doc_id: Identifier of the document to create or replace.
+            doc: Document data or an instance of the configured ODM model.
+            index: Explicit index to target. When omitted, uses the first
+                configured index.
+        """
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"index": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps(saved_doc),
+            )
+        )
+
+    def add_upsert_operation(self, doc_id, doc, index=None):
+        """Queue an update operation that creates the document when absent.
+
+        The operation uses Elasticsearch's ``doc_as_upsert`` option. Its
+        serialized document includes ``id`` set to ``doc_id``.
+
+        Args:
+            doc_id: Identifier of the document to update or create.
+            doc: Document data or an instance of the configured ODM model.
+            index: Explicit index to target. When omitted, uses the first
+                configured index.
+        """
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"update": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps({"doc": saved_doc, "doc_as_upsert": True}),
+            )
+        )
+
+    def add_update_operation(self, doc_id, doc, index=None, fields: Optional[List[str]] = None):
+        """Queue a partial-merge Elasticsearch ``update`` operation.
+
+        Args:
+            doc_id: Identifier of the document to update.
+            doc: Document data or an instance of the configured ODM model.
+            index: Explicit index to target. When omitted, queues one update
+                operation for each configured index.
+
+            fields: When provided, restricts the update body to just these dotted field
+                paths (any valid key from :meth:`ESCollection.fields`/`Model.flat_fields`),
+                dropping everything else so unrelated concurrent edits to the stored
+                document aren't clobbered. Entries may contain ``*`` wildcards, expanded
+                against the model's known field paths (e.g. ``"items.*"`` selects every
+                subfield of the ``items`` list without touching ``title``, ``summary``,
+                etc.). A path with no wildcard is kept wholesale, including any nested
+                subfields, e.g. ``"items"`` alone also keeps every subfield of ``items``.
+        """
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc, mask=list(doc.keys())).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+
+        if fields is not None:
+            saved_doc = prune_to_paths(saved_doc, allowed=expand_field_patterns(self.model, fields))
+
+        if index:
+            self.operations.append(
+                (
+                    json.dumps({"update": {"_index": index, "_id": doc_id}}),
+                    json.dumps({"doc": saved_doc}),
+                )
+            )
+        else:
+            for cur_index in self.indexes:
+                self.operations.append(
+                    (
+                        json.dumps({"update": {"_index": cur_index, "_id": doc_id}}),
+                        json.dumps({"doc": saved_doc}),
+                    )
+                )
+
+    def get_plan_data(self):
+        """Render all queued operations as an Elasticsearch bulk request.
+
+        Returns:
+            Newline-delimited JSON with a final trailing newline, ready to send
+            to Elasticsearch's ``_bulk`` endpoint.
+        """
+        return self._get_plan_for_operations()
+
+    def get_plan_batches(self, batch_size: int | None = DEFAULT_BATCH_SIZE):
+        """Yield queued operations as separate Elasticsearch bulk payloads.
+
+        Args:
+            batch_size: Maximum number of queued operations in each payload.
+                Pass ``None`` to produce one payload containing all operations.
+
+        Yields:
+            Newline-delimited JSON payloads, each with a trailing newline.
+        """
+        if batch_size is None:
+            batch_size = len(self.operations)
+        for ptr in range(0, len(self.operations), batch_size):
+            yield self._get_plan_for_operations(self.operations[ptr : ptr + batch_size])
+
+    def _flatten_operations(self, batch: List[_OPERATION_GROUP] | None = None) -> List[str]:
+        """Flatten queued action/body groups into their NDJSON lines.
+
+        Args:
+            batch: Operation groups to flatten. Uses every queued operation
+                when omitted or empty.
+
+        Returns:
+            JSON action and document lines in bulk API order.
+        """
+        if not batch:
+            batch = self.operations
+
+        flattened: list[str] = []
+        for op in batch:
+            flattened.extend(op)
+
+        return flattened
+
+    def _get_plan_for_operations(self, batch: List[_OPERATION_GROUP] | None = None) -> str:
+        """Render operation groups as a newline-delimited JSON payload.
+
+        Args:
+            batch: Operation groups to render. Uses every queued operation when
+                omitted or empty.
+
+        Returns:
+            The rendered payload with a final trailing newline.
+
+        Warns:
+            UserWarning: If the encoded payload exceeds the configured maximum
+                Elasticsearch request size.
+        """
+        plan = "\n".join(self._flatten_operations(batch)) + "\n"
+
+        if ELASTIC_MAX_REQUEST_SIZE and len(plan.encode("utf-8")) > ELASTIC_MAX_REQUEST_SIZE:
+            warnings.warn(
+                f"Bulk plan exceeds maximum request size of {ELASTIC_MAX_REQUEST_SIZE} bytes. "
+                f"Current size: {len(plan.encode('utf-8'))} bytes."
+            )
+
+        return plan

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/howler/datastore/collection.py

@@ -40,6 +40,7 @@ from howler.datastore.support.schemas import (
     default_mapping,
 )
 from howler.datastore.types import AggSearchResult, SearchResult
+from howler.datastore.utils import expand_field_patterns
 from howler.odm.base import (
     BANNED_FIELDS,
     IP,
@@ -565,7 +566,14 @@ class ESCollection(Generic[ModelType]):
         for operation_batch in operations.get_plan_batches():
             response = self.with_retries(self.datastore.client.bulk, operations=operation_batch, refresh=refresh)
             responses.append(response)
-        return not any(response["errors"] for response in responses)
+
+        has_errors = False
+        for response in responses:
+            if response["errors"]:
+                has_errors = True
+                logger.error("Errors on bulk plan: %s", response["errors"])
+
+        return not has_errors
 
     def get_bulk_plan(self):
         """
@@ -1628,23 +1636,8 @@ class ESCollection(Generic[ModelType]):
         if not self.model_class or "*" not in fl:
             return fl
 
-        all_fields = list(self.model_class.flat_fields().keys())
-        expanded: list[str] = []
-        for pattern in fl.split(","):
-            pattern = pattern.strip()
-            if not pattern:
-                # Skip empty entries (e.g. from trailing commas).
-                continue
-            if "*" not in pattern or pattern == "*":
-                # Exact names and the bare '*' (meaning "all fields") are kept as-is.
-                expanded.append(pattern)
-            else:
-                # Convert the glob-style wildcard to a full regex pattern.
-                # Replace '*' with '.*' and escape all other regex special characters.
-                regex = re.compile("^" + re.escape(pattern).replace(r"\*", ".*") + "$")
-                matched = [f for f in all_fields if regex.match(f)]
-                expanded.extend(matched if matched else [pattern])
-        return ",".join(expanded)
+        patterns = [p.strip() for p in fl.split(",") if p.strip()]
+        return ",".join(sorted(expand_field_patterns(self.model_class, patterns, preserve_all=True)))
 
     def _format_output(self, result, fields=None, as_obj=True):
         # Getting search document data

howler_api-4.1.0.dev1387/howler/datastore/utils.py

@@ -0,0 +1,59 @@
+"""Shared helpers for building field-scoped Elasticsearch operations from ODM models."""
+
+import re
+from typing import Iterable, Optional
+
+from howler import odm
+
+
+def expand_field_patterns(
+    model: Optional[type[odm.Model]], patterns: Iterable[str], preserve_all: bool = False
+) -> set[str]:
+    """Expand `*`-wildcard field patterns against a model's known dotted field paths.
+
+    Entries without a `*` are kept as-is (even if the model doesn't recognize them, so
+    synthetic keys like `__non_doc_raw__` still work). Without a model, patterns can't be
+    expanded and are kept literally.
+    """
+    if model is None:
+        return set(patterns)
+
+    known_paths = list(model.flat_fields().keys())
+    expanded: set[str] = set()
+    for pattern in patterns:
+        if "*" not in pattern:
+            expanded.add(pattern)
+            continue
+
+        if pattern == "*" and preserve_all:
+            expanded.add(pattern)
+            continue
+
+        regex = re.compile("^" + re.escape(pattern).replace(r"\*", ".*") + "$")
+        matches = [path for path in known_paths if regex.match(path)]
+        expanded.update(matches)
+    return expanded
+
+
+def prune_to_paths(value, allowed: set[str], prefix: str = ""):
+    """Recursively keep only the parts of `value` selected by dotted paths in `allowed`.
+
+    A key is kept wholesale (subtree included as-is) if its own path is in `allowed`;
+    otherwise it's kept and recursed into if some allowed path starts with it, so
+    e.g. `allowed={"items.type"}` prunes each entry of a list field named `items` down
+    to just its `type` subfield.
+    """
+    if isinstance(value, dict):
+        result = {}
+        for key, sub_value in value.items():
+            cur_path = f"{prefix}.{key}" if prefix else key
+            if cur_path in allowed:
+                result[key] = sub_value
+            elif any(path.startswith(f"{cur_path}.") for path in allowed):
+                result[key] = prune_to_paths(sub_value, allowed, cur_path)
+        return result
+
+    if isinstance(value, list):
+        return [prune_to_paths(entry, allowed, prefix) for entry in value]
+
+    return value

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/howler/remote/datatypes/__init__.py

@@ -139,7 +139,7 @@ def get_client(
     host: str | redis.Redis | redis.StrictRedis | redis.RedisCluster | None,
     port: int | None,
     private: bool = False,
-):
+) -> redis.Redis | redis.StrictRedis | redis.RedisCluster:
     """
     Get Redis instance.
 

{howler_api-4.1.0.dev1368 → howler_api-4.1.0.dev1387}/howler/remote/datatypes/queues/named.py

@@ -1,6 +1,8 @@
 import json
 import time
-from typing import Any, Generic, Optional, TypeVar
+from typing import Generic, Optional, TypeVar
+
+import redis
 
 from howler.common.exceptions import HowlerTypeError
 from howler.remote.datatypes import get_client, retry_call
@@ -10,7 +12,7 @@ T = TypeVar("T")
 
 class NamedQueue(Generic[T]):
     def __init__(self, name: str, host=None, port=None, private: bool = False, ttl: int = 0):
-        self.c: Any = get_client(host, port, private)
+        self.c: redis.Redis | redis.StrictRedis | redis.RedisCluster = get_client(host, port, private)
         self.name: str = name
         self.ttl: int = ttl
         self.last_expire_time: float = 0
@@ -66,8 +68,7 @@ class NamedQueue(Generic[T]):
             return json.loads(response)
 
     def push(self, *messages: T):
-        for message in messages:
-            retry_call(self.c.rpush, self.name, json.dumps(message))
+        retry_call(self.c.rpush, self.name, *[json.dumps(message) for message in messages])
         self._conditional_expire()
 
     def unpop(self, *messages: T):