howler-api 4.0.0.dev1299__tar.gz → 4.0.0.dev1311__tar.gz

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1299
+Version: 4.0.0.dev1311
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/api/v1/hit.py

@@ -31,6 +31,7 @@ from howler.odm.models.howler_data import Comment, HitOperationType, HitStatusTr
 from howler.odm.models.user import User
 from howler.security import api_login
 from howler.services import action_service, analytic_service, comms_service, hit_service
+from howler.utils.constants import DEBUG_FORCE_REFRESH
 from howler.utils.str_utils import sanitize_lucene_query
 
 MAX_COMMENT_LEN = 5000
@@ -60,7 +61,8 @@ def create_hits(user: User, **kwargs):
     None
 
     Optional Arguments:
-    refresh =>  ('true') Whether to refresh (commit) the datastore before returning.
+    refresh =>  ('true' | 'false' | 'wait_for') Whether to refresh the datastore before returning.
+        'wait_for' will wait for the change to be visible in search.
 
     Data Block:
     {
@@ -121,17 +123,12 @@ def create_hits(user: User, **kwargs):
 
     if len(response_body["invalid"]) == 0:
         if len(odms) > 0:
-            for odm in odms:  # Save all but the last hit without refreshing
+            for odm in odms:
                 # Ensure all ids are consistent
                 if odm.event is not None:
                     odm.event.id = odm.howler.id
-                # TODO keeping the commit approach until we can batch create hits
-                # passing refresh to each one is potentially even more inefficient
-                hit_service.create_hit(odm.howler.id, odm, user=user.uname, refresh="false")
-
-            if refresh == "true":
-                datastore().hit.commit()
 
+            hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
             analytic_service.save_from_hits(odms, user, refresh=refresh)
             action_service.enqueue_action_execution([odm.howler.id for odm in odms], trigger="create", user=user)
 
@@ -577,7 +574,7 @@ def add_label(id, label_set, user, **kwargs):
         id,
         [hit_helper.list_add(f"howler.labels.{label_set}", label) for label in labels],
         user["uname"],
-        refresh=refresh,
+        refresh="true" if DEBUG_FORCE_REFRESH else refresh,
     )
 
     action_service.enqueue_action_execution([id], trigger="add_label", user=user)
@@ -634,7 +631,7 @@ def remove_labels(id, label_set, user, **kwargs):
         id,
         [hit_helper.list_remove(f"howler.labels.{label_set}", label) for label in labels],
         user["uname"],
-        refresh=refresh,
+        refresh="true" if DEBUG_FORCE_REFRESH else refresh,
     )
 
     action_service.enqueue_action_execution([id], trigger="remove_label", user=user)
@@ -1042,7 +1039,8 @@ def create_bundle(user: User, **kwargs):
     None
 
     Optional Arguments:
-    refresh =>  ('true') Whether to refresh (commit) the datastore before returning.
+    refresh =>  ('true' | 'false' | 'wait_for') Whether to refresh the datastore before returning.
+        'wait_for' will wait for the change to be visible in search.
 
     Data Block:
     {

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/api/v1/tool.py

@@ -3,6 +3,7 @@ from typing import Any, Optional
 from flask import request
 
 from howler.api import bad_request, created, make_subapi_blueprint
+from howler.api.v1.utils.params import parse_parameters, parse_refresh
 from howler.common.exceptions import HowlerException
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
@@ -13,6 +14,7 @@ from howler.odm.models.hit import Hit
 from howler.odm.models.user import User
 from howler.security import api_login
 from howler.services import action_service, analytic_service, hit_service
+from howler.utils.constants import DEBUG_FORCE_REFRESH
 from howler.utils.dict_utils import flatten
 from howler.utils.isotime import now_as_iso
 from howler.utils.str_utils import get_parent_key
@@ -30,6 +32,7 @@ hit_helper = OdmHelper(Hit)
 @generate_swagger_docs()
 @tool_api.route("/<tool_name>/hits", methods=["POST", "PUT"])
 @api_login(required_priv=["W"])
+@parse_parameters(refresh=parse_refresh)
 def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
     """Create one or many hits for a tool using field mapping.
 
@@ -39,6 +42,10 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
     Arguments:
     None
 
+    Optional Arguments:
+    refresh =>  ('true' | 'false' | 'wait_for') Whether to refresh the datastore before returning.
+        'wait_for' will wait for the change to be visible in search.
+
     Data Block:
     {
         "map": {            # For each field in the hit, list of field data will be copied to
@@ -64,6 +71,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
         Use POST /api/v1/hit/ directly with pre-mapped hit data instead.
     """
     data = request.json
+    refresh = kwargs.get("refresh", None)
     if not isinstance(data, dict):
         return bad_request(err="Invalid data format")
 
@@ -204,10 +212,9 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
         if bundle_odm is not None:
             child_odms = [odm for odm in odms if odm.howler.id != bundle_id]
 
-            for odm in child_odms:
-                hit_service.create_hit(odm.howler.id, odm, user=user.uname)
+            hit_service.create_hits(child_odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
+            analytic_service.save_from_hits(odms, user, refresh=refresh)
 
-            analytic_service.save_from_hits(odms, user)
             child_ids = [odm.howler.id for odm in child_odms]
             bundle_data = bundle_odm.as_primitives()
             result = bundle_compat_service.create_bundle(bundle_data, child_ids, user=user, refresh="wait_for")
@@ -220,15 +227,11 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
                     entry["_case_id"] = result.get("_case_id")
         else:
             # The bundle hit may have been dropped by de-duplication; ingest remaining hits directly.
-            for odm in odms:
-                hit_service.create_hit(odm.howler.id, odm, user=user.uname)
-            analytic_service.save_from_hits(odms, user)
+            hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
+            analytic_service.save_from_hits(odms, user, refresh=refresh)
     else:
-        for odm in odms:
-            hit_service.create_hit(odm.howler.id, odm, user=user.uname)
-        analytic_service.save_from_hits(odms, user)
-
-    datastore().hit.commit()
+        hit_service.create_hits(odms, user=user.uname, refresh="true" if DEBUG_FORCE_REFRESH else refresh)
+        analytic_service.save_from_hits(odms, user, refresh=refresh)
 
     action_service.enqueue_action_execution([entry["id"] for entry in out], trigger="create", user=user)
 

howler_api-4.0.0.dev1311/howler/datastore/bulk.py

@@ -0,0 +1,148 @@
+import json
+import warnings
+from copy import deepcopy
+from typing import List, Optional
+
+from howler import odm
+from howler.config import config
+
+_OPERATION_GROUP = tuple[str] | tuple[str, str]
+
+ELASTIC_MAX_REQUEST_SIZE = config.datastore.max_request_size
+DEFAULT_BATCH_SIZE = config.datastore.request_batch_size
+
+
+class ElasticBulkPlan(object):
+    def __init__(self, indexes: List[str], model: Optional[type[odm.Model]] = None):
+        self.indexes = indexes
+        self.model = model
+        self.operations: list[_OPERATION_GROUP] = []
+
+    @property
+    def empty(self):
+        return len(self.operations) == 0
+
+    def add_delete_operation(self, doc_id, index=None):
+        if index:
+            self.operations.append((json.dumps({"delete": {"_index": index, "_id": doc_id}}),))
+        else:
+            for cur_index in self.indexes:
+                self.operations.append((json.dumps({"delete": {"_index": cur_index, "_id": doc_id}}),))
+
+    def add_insert_operation(self, doc_id: str, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"create": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps(saved_doc),
+            )
+        )
+
+    def add_index_operation(self, doc_id, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"index": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps(saved_doc),
+            )
+        )
+
+    def add_upsert_operation(self, doc_id, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(
+            (
+                json.dumps({"update": {"_index": index or self.indexes[0], "_id": doc_id}}),
+                json.dumps({"doc": saved_doc, "doc_as_upsert": True}),
+            )
+        )
+
+    def add_update_operation(self, doc_id, doc, index=None):
+        if self.model and isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc, mask=list(doc.keys())).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+
+        if index:
+            self.operations.append(
+                (
+                    json.dumps({"update": {"_index": index, "_id": doc_id}}),
+                    json.dumps({"doc": saved_doc}),
+                )
+            )
+        else:
+            for cur_index in self.indexes:
+                self.operations.append(
+                    (
+                        json.dumps({"update": {"_index": cur_index, "_id": doc_id}}),
+                        json.dumps({"doc": saved_doc}),
+                    )
+                )
+
+    def get_plan_data(self):
+        """Construct the bulk request from the current operations"""
+        return self._get_plan_for_operations()
+
+    def get_plan_batches(self, batch_size: int | None = DEFAULT_BATCH_SIZE):
+        """Yield plan data in batches"""
+        if batch_size is None:
+            batch_size = len(self.operations)
+        for ptr in range(0, len(self.operations), batch_size):
+            yield self._get_plan_for_operations(self.operations[ptr : ptr + batch_size])
+
+    def _flatten_operations(self, batch: List[_OPERATION_GROUP] | None = None) -> List[str]:
+        """Flatten the (operation, data) tuples for a batch or the full list of operations if no batch provided"""
+        if not batch:
+            batch = self.operations
+
+        flattened: list[str] = []
+        for op in batch:
+            flattened.extend(op)
+
+        return flattened
+
+    def _get_plan_for_operations(self, batch: List[_OPERATION_GROUP] | None = None) -> str:
+        """Get the bulk plan string for a batch or the full list of operations if no batch provided"""
+        plan = "\n".join(self._flatten_operations(batch)) + "\n"
+
+        if ELASTIC_MAX_REQUEST_SIZE and len(plan.encode("utf-8")) > ELASTIC_MAX_REQUEST_SIZE:
+            warnings.warn(
+                f"Bulk plan exceeds maximum request size of {ELASTIC_MAX_REQUEST_SIZE} bytes. "
+                f"Current size: {len(plan.encode('utf-8'))} bytes."
+            )
+
+        return plan

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/datastore/collection.py

@@ -22,6 +22,7 @@ from howler import odm
 from howler.common.exceptions import HowlerRuntimeError, HowlerValueError, NonRecoverableError
 from howler.common.loader import DATASTORE_INDEX_PREFIX
 from howler.common.logging.format import HWL_DATE_FORMAT, HWL_LOG_FORMAT
+from howler.datastore.bulk import ElasticBulkPlan
 from howler.datastore.constants import BACK_MAPPING, TYPE_MAPPING
 from howler.datastore.exceptions import (
     DataStoreException,
@@ -251,19 +252,50 @@ class ESCollection(Generic[ModelType]):
 
     @property
     def index_list_full(self):
+        """Return every physical index for this collection, newest ILM index first.
+
+        Maintenance operations that must visit historical rollover indexes should use this
+        property. The active index is included even when no ILM rollover index exists.
+        """
         if not self._index_list:
-            self._index_list = list(self.with_retries(self.datastore.client.indices.get, index=f"{self.name}-*").keys())
+            self._index_list = self._get_ilm_index_names()
 
-        return [self.index_name] + sorted(self._index_list, reverse=True)
+        return list(dict.fromkeys([self.index_name, *sorted(self._index_list, reverse=True)]))
 
     @property
     def index_list(self):
-        """This property contains the list of valid indexes for the current collection.
+        """Return the active physical write index for this collection.
 
-        :return: list of valid indexes for this collection
+        Historical ILM rollover indexes are intentionally excluded. Bulk plans use this
+        list for default write, update, and delete targets, and widening it would fan those
+        operations out across historical indexes.
         """
         return [self.index_name]
 
+    def _get_ilm_index_names(self) -> list[str]:
+        """Return physical ILM rollover indexes, excluding temporary maintenance indexes."""
+        try:
+            indices = self.datastore.client.indices.get(
+                index=f"{self.name}-0*",
+                ignore_unavailable=True,
+            )
+        except elasticsearch.exceptions.NotFoundError:
+            return []
+
+        index_pattern = re.compile(rf"^{re.escape(self.name)}-\d{{6}}$")
+        return sorted(index for index in indices if index_pattern.fullmatch(index))
+
+    def _refresh_ilm_index_name(self):
+        """Set ``index_name`` to the latest existing ILM index, when one exists."""
+        if not self.ilm_config:
+            return
+
+        ilm_indices = self._get_ilm_index_names()
+
+        if ilm_indices:
+            self._index_list = ilm_indices
+            self.index_name = ilm_indices[-1]
+
     def scan_with_retry(
         self,
         query,
@@ -523,6 +555,26 @@ class ESCollection(Generic[ModelType]):
             else:
                 updated += res["updated"]
 
+    def bulk(self, operations: ElasticBulkPlan, refresh: str | None = None):
+        """
+        Execute a bulk plan.
+
+        :return: True if the operation completed without errors
+        """
+        responses = []
+        for operation_batch in operations.get_plan_batches():
+            response = self.with_retries(self.datastore.client.bulk, operations=operation_batch, refresh=refresh)
+            responses.append(response)
+        return not any(response["errors"] for response in responses)
+
+    def get_bulk_plan(self):
+        """
+        Create a BulkPlan tailored for the current datastore
+
+        :return: The BulkPlan object
+        """
+        return ElasticBulkPlan(self.index_list, self.model_class)
+
     @tracer.start_as_current_span(f"{__name__}.commit")
     def commit(self):
         """This function should be overloaded to perform a commit of the index data of all the different hosts
@@ -709,7 +761,7 @@ class ESCollection(Generic[ModelType]):
     def reindex(self, allow_failures: bool = False, request_timeout: int = 60):
         """Reindex all the data of the collection into a freshly mapped index.
 
-        For every index in ``self.index_list`` the data is copied into a temporary
+        For every physical index in ``self.index_list_full`` the data is copied into a temporary
         ``__reindex`` index that uses the current mappings/settings. Writes to the source
         index are blocked while the copy runs so the reindex result and document counts can
         be validated before the original index is deleted. The temporary index is only
@@ -729,7 +781,8 @@ class ESCollection(Generic[ModelType]):
             operation can be recovered with :meth:`reindex_cleanup`.
         """
         logger.warning("Beginning Reindex")
-        for index in self.index_list:
+        self._refresh_ilm_index_name()
+        for index in self.index_list_full:
             new_name = f"{index}__reindex"
             index_data = None
             source_count = None
@@ -777,7 +830,7 @@ class ESCollection(Generic[ModelType]):
                     self.datastore.client.indices.create,
                     index=new_name,
                     mappings=self._get_index_mappings(),
-                    settings=self._get_index_settings(),
+                    settings=self._get_reindex_settings(index_data),
                 )
 
                 # Reindex data into target
@@ -826,12 +879,10 @@ class ESCollection(Generic[ModelType]):
                     )
 
                     alias_actions = []
-                    aliases = index_data.get("aliases", {}) or {self.name: {"is_write_index": True}}
+                    aliases = index_data.get("aliases", {})
                     for alias, alias_data in aliases.items():
                         alias_action = {"index": index, "alias": alias}
                         alias_action.update(alias_data)
-                        if alias == self.name or alias_data.get("is_write_index", False):
-                            alias_action["is_write_index"] = True
                         alias_actions.append({"add": alias_action})
                     alias_actions.append({"remove_index": {"index": new_name}})
                     self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
@@ -859,6 +910,14 @@ class ESCollection(Generic[ModelType]):
 
         return True
 
+    def _get_reindex_settings(self, index_data: dict) -> dict:
+        """Build settings for a reindex target without dropping ILM lifecycle metadata."""
+        settings = self._get_index_settings()
+        lifecycle = index_data.get("settings", {}).get("index", {}).get("lifecycle")
+        if lifecycle:
+            settings["index"]["lifecycle"] = deepcopy(lifecycle)
+        return settings
+
     def _validate_reindex_result(self, index, new_name, reindex_result, allow_failures):
         """Validate the result of an Elasticsearch reindex task.
 
@@ -935,17 +994,18 @@ class ESCollection(Generic[ModelType]):
     def reindex_cleanup(self):
         """Recover from a failed or interrupted :meth:`reindex` run.
 
-        For every index that still has a leftover ``__reindex`` index, restore the source
-        index as the writable target and delete the orphaned ``__reindex`` index. If the
-        source index is missing, raise an error instead of deleting ``__reindex`` because
-        it may be the only remaining copy of the data.
+        For every physical index that still has a leftover ``__reindex`` index, restore the source
+        index's original aliases and delete the orphaned ``__reindex`` index. If the source
+        index is missing, raise an error instead of deleting ``__reindex`` because it may
+        be the only remaining copy of the data.
 
         Write blocks set during reindexing are cleared so the collection is usable again.
 
         :return: ``True`` when cleanup completed on all indexes
         """
         logger.warning("Beginning reindex cleanup")
-        for index in self.index_list:
+        self._refresh_ilm_index_name()
+        for index in self.index_list_full:
             new_name = f"{index}__reindex"
 
             if not self.with_retries(self.datastore.client.indices.exists, index=new_name):
@@ -966,17 +1026,13 @@ class ESCollection(Generic[ModelType]):
             reindex_aliases = new_index_data.get("aliases", {})
 
             alias_actions = []
-            for alias in sorted(set(source_aliases) | set(reindex_aliases) | {self.name}):
+            for alias in sorted(set(source_aliases) | set(reindex_aliases)):
                 source_alias_data = source_aliases.get(alias, {})
                 reindex_alias_data = reindex_aliases.get(alias, {})
 
                 add_alias_data = {"index": index, "alias": alias}
                 add_alias_data.update(source_alias_data)
-                if (
-                    alias == self.name
-                    or source_alias_data.get("is_write_index", False)
-                    or reindex_alias_data.get("is_write_index", False)
-                ):
+                if source_alias_data.get("is_write_index", False) or reindex_alias_data.get("is_write_index", False):
                     add_alias_data["is_write_index"] = True
 
                 alias_actions.append({"add": add_alias_data})
@@ -2884,7 +2940,42 @@ class ESCollection(Generic[ModelType]):
         if existing_ilm_indices:
             # ILM already bootstrapped — ensure the alias exists
             latest = sorted(existing_ilm_indices)[-1]
-            if not self.with_retries(self.datastore.client.indices.exists_alias, name=self.name):
+            legacy_hot_index = f"{self.name}_hot"
+            if self.with_retries(self.datastore.client.indices.exists_alias, name=self.name):
+                alias_indices = self.with_retries(self.datastore.client.indices.get_alias, name=self.name)
+                alias_actions = []
+                for alias_index, alias_index_data in alias_indices.items():
+                    alias_data = alias_index_data.get("aliases", {}).get(self.name, {})
+                    if alias_index == legacy_hot_index:
+                        alias_actions.append({"remove": {"index": alias_index, "alias": self.name}})
+                    elif alias_index != latest and alias_data.get("is_write_index", False):
+                        alias_actions.append(
+                            {
+                                "add": {
+                                    "index": alias_index,
+                                    "alias": self.name,
+                                    **alias_data,
+                                    "is_write_index": False,
+                                }
+                            }
+                        )
+
+                latest_alias_data = alias_indices.get(latest, {}).get("aliases", {}).get(self.name, {})
+                if not latest_alias_data.get("is_write_index", False):
+                    alias_actions.append(
+                        {
+                            "add": {
+                                "index": latest,
+                                "alias": self.name,
+                                **latest_alias_data,
+                                "is_write_index": True,
+                            }
+                        }
+                    )
+
+                if alias_actions:
+                    self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
+            else:
                 # Find the latest index to set as write index
                 self.with_retries(
                     self.datastore.client.indices.put_alias,

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/external/reindex_data.py

@@ -99,6 +99,7 @@ if __name__ == "__main__":
 
     for index_name in selected:
         collection: ESCollection = getattr(ds, index_name)
+        collection._refresh_ilm_index_name()
 
         if args.verbose:
             print(f"Index schema for '{index_name}':")

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/odm/base.py

@@ -1358,6 +1358,9 @@ class Model:
         if data is None:
             data = {}
 
+        if "__index" in data:
+            data.pop("__index")  # Ignore the __index field sometimes generated by as_primitives
+
         if not hasattr(data, "items"):
             raise HowlerTypeError(f"'{self.__class__.__name__}' object must be constructed with dict like")
         self._odm_py_obj = {}

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/odm/models/config.py

@@ -149,7 +149,15 @@ class Datastore(BaseModel):
     """
 
     hosts: list[Host] = Field(
-        default=[Host(name="elastic", username="elastic", password="devpass", scheme="http", host="localhost:9200")],  # noqa: S106
+        default=[
+            Host(
+                name="elastic",
+                username="elastic",
+                password="devpass",  # noqa: S106
+                scheme="http",
+                host="localhost:9200",
+            )
+        ],
         description="List of hosts used for the datastore",
     )
     type: Literal["elasticsearch"] = Field(
@@ -159,6 +167,8 @@ class Datastore(BaseModel):
         default_factory=ILMConfig,
         description="Index Lifecycle Management configuration",
     )
+    max_request_size: Optional[int] = Field(description="Maximum request size in bytes", default=100_000_000)  # 100MB
+    request_batch_size: Optional[int] = Field(description="Default number of operations for bulk requests", default=500)
 
 
 class Logging(BaseModel):

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/odm/random_data.py

@@ -867,8 +867,12 @@ def wipe_actions(ds: HowlerDatastore):
 def create_dossiers(ds: HowlerDatastore, num_dossiers: int = 5):
     "Create random dossiers"
     users = ds.user.search("*:*")["items"]
-    for _ in range(num_dossiers):
+    for index in range(num_dossiers):
         dossier = generate_useful_dossier(users)
+        if index == 0:
+            dossier.type = "personal"
+        elif index == 1:
+            dossier.type = "global"
         ds.dossier.save(dossier.dossier_id, dossier)
 
     ds.dossier.commit()

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/services/analytic_service.py

@@ -123,11 +123,11 @@ def save_from_hits(
             analytics.append(analytic_update)
 
     if analytics:
-        for analytic in analytics[:-1]:
-            storage.analytic.save(analytic.analytic_id, analytic)
+        bulk_plan = storage.analytic.get_bulk_plan()
+        for analytic in analytics:
+            bulk_plan.add_index_operation(analytic.analytic_id, analytic)
 
-        # save the last one passing on the refresh parameter
-        storage.analytic.save(analytics[-1].analytic_id, analytics[-1], refresh=refresh)
+        storage.analytic.bulk(bulk_plan, refresh=refresh)
 
 
 def _get_analytic_updates_from_hit_group(analytic_name: str, hit_group: list[Hit], user: User) -> Analytic | None:

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/services/hit_service.py

@@ -442,6 +442,31 @@ def create_hit(
     return datastore().hit.save(id, hit, refresh=refresh)
 
 
+@tracer.start_as_current_span(f"{__name__}.create_hits")
+def create_hits(
+    hits: list[Hit], user: Optional[str] = None, overwrite: bool = False, refresh: str | None = None
+) -> bool:
+    """Bulk create multiple hits in the database.
+
+    Similar to create_hit for batch.
+    Will raise and abort the entire batch if any hit already exists and overwrite=False.
+    """
+    storage = datastore()
+    bulk_plan = storage.hit.get_bulk_plan()
+
+    for hit in hits:
+        if not overwrite and storage.hit.exists(hit.howler.id):
+            raise ResourceExists("Hit %s already exists in datastore" % hit.howler.id)
+
+        if user:
+            hit.howler.log = [Log({"timestamp": "NOW", "explanation": "Created hit", "user": user})]
+
+        CREATED_HITS.labels(hit.howler.analytic).inc()
+        bulk_plan.add_insert_operation(hit.howler.id, hit)
+
+    return storage.hit.bulk(bulk_plan, refresh=refresh)
+
+
 @tracer.start_as_current_span(f"{__name__}.update_hit")
 def update_hit(
     hit_id: str,
@@ -476,6 +501,21 @@ def update_hit(
     return _update_hit(hit_id, operations, user, version=version, refresh=refresh)
 
 
+@tracer.start_as_current_span(f"{__name__}.overwrite_hits")
+def overwrite_hits(hits: list[Hit], refresh: str | None = None) -> bool:
+    """Bulk save multiple hits to the datastore.
+
+    Similar to save_hit for batch without versioning. Will overwrite existing hits with the same ID.
+    """
+    storage = datastore()
+    bulk_plan = storage.hit.get_bulk_plan()
+
+    for hit in hits:
+        bulk_plan.add_index_operation(hit.howler.id, hit)
+
+    return storage.hit.bulk(bulk_plan, refresh=refresh)
+
+
 @typing.no_type_check
 @tracer.start_as_current_span(f"{__name__}.save_hit")
 def save_hit(hit: Hit, version: Optional[str] = None, refresh: str | None = None) -> tuple[Hit, str]:

{howler_api-4.0.0.dev1299 → howler_api-4.0.0.dev1311}/howler/utils/constants.py

@@ -2,3 +2,4 @@ import os
 import sys
 
 TESTING = os.getenv("TESTING", "false").lower() in ["true", "1", "yes"] or "pytest" in sys.modules
+DEBUG_FORCE_REFRESH = os.getenv("DEBUG_FORCE_REFRESH", "false").lower() in ["true", "1", "yes"] or TESTING