howler-api 4.0.0.dev1298__tar.gz → 4.0.0.dev1308__tar.gz

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1298
+Version: 4.0.0.dev1308
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/api/v1/hit.py

@@ -31,6 +31,7 @@ from howler.odm.models.howler_data import Comment, HitOperationType, HitStatusTr
 from howler.odm.models.user import User
 from howler.security import api_login
 from howler.services import action_service, analytic_service, comms_service, hit_service
+from howler.utils.constants import DEBUG_FORCE_REFRESH
 from howler.utils.str_utils import sanitize_lucene_query
 
 MAX_COMMENT_LEN = 5000
@@ -60,7 +61,8 @@ def create_hits(user: User, **kwargs):
     None
 
     Optional Arguments:
-    refresh =>  ('true') Whether to refresh (commit) the datastore before returning.
+    refresh =>  ('true' | 'false' | 'wait_for') Whether to refresh the datastore before returning.
+        'wait_for' will wait for the change to be visible in search.
 
     Data Block:
     {
@@ -121,17 +123,12 @@ def create_hits(user: User, **kwargs):
 
     if len(response_body["invalid"]) == 0:
         if len(odms) > 0:
-            for odm in odms:  # Save all but the last hit without refreshing
+            for odm in odms:
                 # Ensure all ids are consistent
                 if odm.event is not None:
                     odm.event.id = odm.howler.id
-                # TODO keeping the commit approach until we can batch create hits
-                # passing refresh to each one is potentially even more inefficient
-                hit_service.create_hit(odm.howler.id, odm, user=user.uname, refresh="false")
-
-            if refresh == "true":
-                datastore().hit.commit()
 
+            hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
             analytic_service.save_from_hits(odms, user, refresh=refresh)
             action_service.enqueue_action_execution([odm.howler.id for odm in odms], trigger="create", user=user)
 
@@ -577,7 +574,7 @@ def add_label(id, label_set, user, **kwargs):
         id,
         [hit_helper.list_add(f"howler.labels.{label_set}", label) for label in labels],
         user["uname"],
-        refresh=refresh,
+        refresh="true" if DEBUG_FORCE_REFRESH else refresh,
     )
 
     action_service.enqueue_action_execution([id], trigger="add_label", user=user)
@@ -634,7 +631,7 @@ def remove_labels(id, label_set, user, **kwargs):
         id,
         [hit_helper.list_remove(f"howler.labels.{label_set}", label) for label in labels],
         user["uname"],
-        refresh=refresh,
+        refresh="true" if DEBUG_FORCE_REFRESH else refresh,
     )
 
     action_service.enqueue_action_execution([id], trigger="remove_label", user=user)
@@ -1042,7 +1039,8 @@ def create_bundle(user: User, **kwargs):
     None
 
     Optional Arguments:
-    refresh =>  ('true') Whether to refresh (commit) the datastore before returning.
+    refresh =>  ('true' | 'false' | 'wait_for') Whether to refresh the datastore before returning.
+        'wait_for' will wait for the change to be visible in search.
 
     Data Block:
     {

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/api/v1/tool.py

@@ -3,6 +3,7 @@ from typing import Any, Optional
 from flask import request
 
 from howler.api import bad_request, created, make_subapi_blueprint
+from howler.api.v1.utils.params import parse_parameters, parse_refresh
 from howler.common.exceptions import HowlerException
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
@@ -13,6 +14,7 @@ from howler.odm.models.hit import Hit
 from howler.odm.models.user import User
 from howler.security import api_login
 from howler.services import action_service, analytic_service, hit_service
+from howler.utils.constants import DEBUG_FORCE_REFRESH
 from howler.utils.dict_utils import flatten
 from howler.utils.isotime import now_as_iso
 from howler.utils.str_utils import get_parent_key
@@ -30,6 +32,7 @@ hit_helper = OdmHelper(Hit)
 @generate_swagger_docs()
 @tool_api.route("/<tool_name>/hits", methods=["POST", "PUT"])
 @api_login(required_priv=["W"])
+@parse_parameters(refresh=parse_refresh)
 def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
     """Create one or many hits for a tool using field mapping.
 
@@ -39,6 +42,10 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
     Arguments:
     None
 
+    Optional Arguments:
+    refresh =>  ('true' | 'false' | 'wait_for') Whether to refresh the datastore before returning.
+        'wait_for' will wait for the change to be visible in search.
+
     Data Block:
     {
         "map": {            # For each field in the hit, list of field data will be copied to
@@ -64,6 +71,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
         Use POST /api/v1/hit/ directly with pre-mapped hit data instead.
     """
     data = request.json
+    refresh = kwargs.get("refresh", None)
     if not isinstance(data, dict):
         return bad_request(err="Invalid data format")
 
@@ -204,10 +212,9 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
         if bundle_odm is not None:
             child_odms = [odm for odm in odms if odm.howler.id != bundle_id]
 
-            for odm in child_odms:
-                hit_service.create_hit(odm.howler.id, odm, user=user.uname)
+            hit_service.create_hits(child_odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
+            analytic_service.save_from_hits(odms, user, refresh=refresh)
 
-            analytic_service.save_from_hits(odms, user)
             child_ids = [odm.howler.id for odm in child_odms]
             bundle_data = bundle_odm.as_primitives()
             result = bundle_compat_service.create_bundle(bundle_data, child_ids, user=user, refresh="wait_for")
@@ -220,15 +227,11 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
                     entry["_case_id"] = result.get("_case_id")
         else:
             # The bundle hit may have been dropped by de-duplication; ingest remaining hits directly.
-            for odm in odms:
-                hit_service.create_hit(odm.howler.id, odm, user=user.uname)
-            analytic_service.save_from_hits(odms, user)
+            hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
+            analytic_service.save_from_hits(odms, user, refresh=refresh)
     else:
-        for odm in odms:
-            hit_service.create_hit(odm.howler.id, odm, user=user.uname)
-        analytic_service.save_from_hits(odms, user)
-
-    datastore().hit.commit()
+        hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
+        analytic_service.save_from_hits(odms, user, refresh=refresh)
 
     action_service.enqueue_action_execution([entry["id"] for entry in out], trigger="create", user=user)
 

howler_api-4.0.0.dev1308/howler/datastore/bulk.py

@@ -0,0 +1,148 @@
+import json
+import warnings
+from copy import deepcopy
+from typing import List, Optional
+
+from howler import odm
+from howler.config import config
+
+_OPERATION_GROUP = tuple[str] | tuple[str, str]
+
+ELASTIC_MAX_REQUEST_SIZE = config.datastore.max_request_size
+DEFAULT_BATCH_SIZE = config.datastore.request_batch_size
+
+
+class ElasticBulkPlan(object):
+    def __init__(self, indexes: List[str], model: Optional[type[odm.Model]] = None):
+        self.indexes = indexes
+        self.model = model
+        self.operations: list[_OPERATION_GROUP] = []
+
+    @property
+    def empty(self):
+        return len(self.operations) == 0
+
+    def add_delete_operation(self, doc_id, index=None):
+        if index:
+            self.operations.append((json.dumps({"delete": {"_index": index, "_id": doc_id}}),))
+        else:
+            for cur_index in self.indexes:
+                self.operations.append((json.dumps({"delete": {"_index": cur_index, "_id": doc_id}}),))
+
+    def add_insert_operation(self, doc_id: str, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"create": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps(saved_doc),
+            )
+        )
+
+    def add_index_operation(self, doc_id, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"index": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps(saved_doc),
+            )
+        )
+
+    def add_upsert_operation(self, doc_id, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"update": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps({"doc": saved_doc, "doc_as_upsert": True}),
+            )
+        )
+
+    def add_update_operation(self, doc_id, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc, mask=list(doc.keys())).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+
+        if index:
+            self.operations.append(
+                (
+                    json.dumps({"update": {"_index": index, "_id": doc_id}}),
+                    json.dumps({"doc": saved_doc}),
+                )
+            )
+        else:
+            for cur_index in self.indexes:
+                self.operations.append(
+                    (
+                        json.dumps({"update": {"_index": cur_index, "_id": doc_id}}),
+                        json.dumps({"doc": saved_doc}),
+                    )
+                )
+
+    def get_plan_data(self):
+        """Construct the bulk request from the current operations"""
+        return self._get_plan_for_operations()
+
+    def get_plan_batches(self, batch_size: int | None = DEFAULT_BATCH_SIZE):
+        """Yield plan data in batches"""
+        if batch_size is None:
+            batch_size = len(self.operations)
+        for ptr in range(0, len(self.operations), batch_size):
+            yield self._get_plan_for_operations(self.operations[ptr : ptr + batch_size])
+
+    def _flatten_operations(self, batch: List[_OPERATION_GROUP] | None = None) -> List[str]:
+        """Flatten the (operation, data) tuples for a batch or the full list of operations if no batch provided"""
+        if not batch:
+            batch = self.operations
+
+        flattened: list[str] = []
+        for op in batch:
+            flattened.extend(op)
+
+        return flattened
+
+    def _get_plan_for_operations(self, batch: List[_OPERATION_GROUP] | None = None) -> str:
+        """Get the bulk plan string for a batch or the full list of operations if no batch provided"""
+        plan = "\n".join(self._flatten_operations(batch)) + "\n"
+
+        if ELASTIC_MAX_REQUEST_SIZE and len(plan.encode("utf-8")) > ELASTIC_MAX_REQUEST_SIZE:
+            warnings.warn(
+                f"Bulk plan exceeds maximum request size of {ELASTIC_MAX_REQUEST_SIZE} bytes. "
+                f"Current size: {len(plan.encode('utf-8'))} bytes."
+            )
+
+        return plan

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/datastore/collection.py

@@ -22,6 +22,7 @@ from howler import odm
 from howler.common.exceptions import HowlerRuntimeError, HowlerValueError, NonRecoverableError
 from howler.common.loader import DATASTORE_INDEX_PREFIX
 from howler.common.logging.format import HWL_DATE_FORMAT, HWL_LOG_FORMAT
+from howler.datastore.bulk import ElasticBulkPlan
 from howler.datastore.constants import BACK_MAPPING, TYPE_MAPPING
 from howler.datastore.exceptions import (
     DataStoreException,
@@ -523,6 +524,26 @@ class ESCollection(Generic[ModelType]):
             else:
                 updated += res["updated"]
 
+    def bulk(self, operations: ElasticBulkPlan, refresh: str | None = None):
+        """
+        Execute a bulk plan.
+
+        :return: True if the operation completed without errors
+        """
+        responses = []
+        for operation_batch in operations.get_plan_batches():
+            response = self.with_retries(self.datastore.client.bulk, operations=operation_batch, refresh=refresh)
+            responses.append(response)
+        return not any(response["errors"] for response in responses)
+
+    def get_bulk_plan(self):
+        """
+        Create a BulkPlan tailored for the current datastore
+
+        :return: The BulkPlan object
+        """
+        return ElasticBulkPlan(self.index_list, self.model_class)
+
     @tracer.start_as_current_span(f"{__name__}.commit")
     def commit(self):
         """This function should be overloaded to perform a commit of the index data of all the different hosts

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/odm/base.py

@@ -1358,6 +1358,9 @@ class Model:
         if data is None:
             data = {}
 
+        if "__index" in data:
+            data.pop("__index")  # Ignore the __index field sometimes generated by as_primitives
+
         if not hasattr(data, "items"):
             raise HowlerTypeError(f"'{self.__class__.__name__}' object must be constructed with dict like")
         self._odm_py_obj = {}

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/odm/models/config.py

@@ -149,7 +149,15 @@ class Datastore(BaseModel):
     """
 
     hosts: list[Host] = Field(
-        default=[Host(name="elastic", username="elastic", password="devpass", scheme="http", host="localhost:9200")],  # noqa: S106
+        default=[
+            Host(
+                name="elastic",
+                username="elastic",
+                password="devpass",  # noqa: S106
+                scheme="http",
+                host="localhost:9200",
+            )
+        ],
         description="List of hosts used for the datastore",
     )
     type: Literal["elasticsearch"] = Field(
@@ -159,6 +167,8 @@ class Datastore(BaseModel):
         default_factory=ILMConfig,
         description="Index Lifecycle Management configuration",
     )
+    max_request_size: Optional[int] = Field(description="Maximum request size in bytes", default=100_000_000)  # 100MB
+    request_batch_size: Optional[int] = Field(description="Default number of operations for bulk requests", default=500)
 
 
 class Logging(BaseModel):

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/services/analytic_service.py

@@ -123,11 +123,11 @@ def save_from_hits(
             analytics.append(analytic_update)
 
     if analytics:
-        for analytic in analytics[:-1]:
-            storage.analytic.save(analytic.analytic_id, analytic)
+        bulk_plan = storage.analytic.get_bulk_plan()
+        for analytic in analytics:
+            bulk_plan.add_index_operation(analytic.analytic_id, analytic)
 
-        # save the last one passing on the refresh parameter
-        storage.analytic.save(analytics[-1].analytic_id, analytics[-1], refresh=refresh)
+        storage.analytic.bulk(bulk_plan, refresh=refresh)
 
 
 def _get_analytic_updates_from_hit_group(analytic_name: str, hit_group: list[Hit], user: User) -> Analytic | None:

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/services/hit_service.py

@@ -442,6 +442,31 @@ def create_hit(
     return datastore().hit.save(id, hit, refresh=refresh)
 
 
+@tracer.start_as_current_span(f"{__name__}.create_hits")
+def create_hits(
+    hits: list[Hit], user: Optional[str] = None, overwrite: bool = False, refresh: str | None = None
+) -> bool:
+    """Bulk create multiple hits in the database.
+
+    Similar to create_hit for batch.
+    Will raise and abort the entire batch if any hit already exists and overwrite=False.
+    """
+    storage = datastore()
+    bulk_plan = storage.hit.get_bulk_plan()
+
+    for hit in hits:
+        if not overwrite and storage.hit.exists(hit.howler.id):
+            raise ResourceExists("Hit %s already exists in datastore" % hit.howler.id)
+
+        if user:
+            hit.howler.log = [Log({"timestamp": "NOW", "explanation": "Created hit", "user": user})]
+
+        CREATED_HITS.labels(hit.howler.analytic).inc()
+        bulk_plan.add_insert_operation(hit.howler.id, hit)
+
+    return storage.hit.bulk(bulk_plan, refresh=refresh)
+
+
 @tracer.start_as_current_span(f"{__name__}.update_hit")
 def update_hit(
     hit_id: str,
@@ -476,6 +501,21 @@ def update_hit(
     return _update_hit(hit_id, operations, user, version=version, refresh=refresh)
 
 
+@tracer.start_as_current_span(f"{__name__}.overwrite_hits")
+def overwrite_hits(hits: list[Hit], refresh: str | None = None) -> bool:
+    """Bulk save multiple hits to the datastore.
+
+    Similar to save_hit for batch without versioning. Will overwrite existing hits with the same ID.
+    """
+    storage = datastore()
+    bulk_plan = storage.hit.get_bulk_plan()
+
+    for hit in hits:
+        bulk_plan.add_index_operation(hit.howler.id, hit)
+
+    return storage.hit.bulk(bulk_plan, refresh=refresh)
+
+
 @typing.no_type_check
 @tracer.start_as_current_span(f"{__name__}.save_hit")
 def save_hit(hit: Hit, version: Optional[str] = None, refresh: str | None = None) -> tuple[Hit, str]:

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/utils/constants.py

@@ -2,3 +2,4 @@ import os
 import sys
 
 TESTING = os.getenv("TESTING", "false").lower() in ["true", "1", "yes"] or "pytest" in sys.modules
+DEBUG_FORCE_REFRESH = os.getenv("DEBUG_FORCE_REFRESH", "false").lower() in ["true", "1", "yes"] or TESTING

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/howler/utils/indexes.py

@@ -1,3 +1,5 @@
+import re
+
 from howler.common.loader import APP_NAME, DATASTORE_INDEX_PREFIX
 from howler.datastore.exceptions import SearchException
 
@@ -53,4 +55,4 @@ def get_logical_index_name(raw_index: str) -> str:
             raw_index = raw_index.removeprefix(f"{index_prefix}-")
             break
 
-    return raw_index.removesuffix("_hot")
+    return re.sub(r"-\d+$", "", raw_index).removesuffix("_hot")

{howler_api-4.0.0.dev1298 → howler_api-4.0.0.dev1308}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev1298"
+version = "4.0.0.dev1308"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",
@@ -255,7 +255,7 @@ typing-extensions = "^4.12.2"
 types-PyYAML = "6.0.12.20260518"
 types-paramiko = "3.5.0.20250801"
 types-pyOpenSSL = "23.3.0.20240106"
-types-six = "1.17.0.20260518"
+types-six = "1.17.0.20260721"
 types-requests = "<2.33.0.20260713"
 websocket-client = "1.9.0"
 types-pytz = "^2024.1.0.20240417"

howler_api-4.0.0.dev1298/howler/datastore/bulk.py

@@ -1,72 +0,0 @@
-import json
-from copy import deepcopy
-
-
-class ElasticBulkPlan(object):
-    def __init__(self, indexes, model=None):
-        self.indexes = indexes
-        self.model = model
-        self.operations = []
-
-    @property
-    def empty(self):
-        return len(self.operations) == 0
-
-    def add_delete_operation(self, doc_id, index=None):
-        if index:
-            self.operations.append(json.dumps({"delete": {"_index": index, "_id": doc_id}}))
-        else:
-            for cur_index in self.indexes:
-                self.operations.append(json.dumps({"delete": {"_index": cur_index, "_id": doc_id}}))
-
-    def add_insert_operation(self, doc_id, doc, index=None):
-        if isinstance(doc, self.model):
-            saved_doc = doc.as_primitives(hidden_fields=True)  # type: ignore
-        elif self.model:
-            saved_doc = self.model(doc).as_primitives(hidden_fields=True)  # type: ignore
-        else:
-            if not isinstance(doc, dict):
-                saved_doc = {"__non_doc_raw__": doc}
-            else:
-                saved_doc = deepcopy(doc)  # type: ignore
-        saved_doc["id"] = doc_id
-
-        self.operations.append(json.dumps({"create": {"_index": index or self.indexes[0], "_id": doc_id}}))
-        self.operations.append(json.dumps(saved_doc))
-
-    def add_upsert_operation(self, doc_id, doc, index=None):
-        if isinstance(doc, self.model):
-            saved_doc = doc.as_primitives(hidden_fields=True)  # type: ignore
-        elif self.model:
-            saved_doc = self.model(doc).as_primitives(hidden_fields=True)  # type: ignore
-        else:
-            if not isinstance(doc, dict):
-                saved_doc = {"__non_doc_raw__": doc}  # type: ignore
-            else:
-                saved_doc = deepcopy(doc)  # type: ignore
-        saved_doc["id"] = doc_id
-
-        self.operations.append(json.dumps({"update": {"_index": index or self.indexes[0], "_id": doc_id}}))
-        self.operations.append(json.dumps({"doc": saved_doc, "doc_as_upsert": True}))
-
-    def add_update_operation(self, doc_id, doc, index=None):
-        if isinstance(doc, self.model):
-            saved_doc = doc.as_primitives(hidden_fields=True)  # type: ignore
-        elif self.model:
-            saved_doc = self.model(doc, mask=list(doc.keys())).as_primitives(hidden_fields=True)  # type: ignore
-        else:
-            if not isinstance(doc, dict):
-                saved_doc = {"__non_doc_raw__": doc}  # type: ignore
-            else:
-                saved_doc = deepcopy(doc)  # type: ignore
-
-        if index:
-            self.operations.append(json.dumps({"update": {"_index": index, "_id": doc_id}}))
-            self.operations.append(json.dumps({"doc": saved_doc}))
-        else:
-            for cur_index in self.indexes:
-                self.operations.append(json.dumps({"update": {"_index": cur_index, "_id": doc_id}}))
-                self.operations.append(json.dumps({"doc": saved_doc}))
-
-    def get_plan_data(self):
-        return "\n".join(self.operations)