howler-api 4.0.3.dev1397__tar.gz → 4.0.4.dev1401__tar.gz

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.3.dev1397
+Version: 4.0.4.dev1401
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/howler/api/v1/action.py

@@ -143,7 +143,7 @@ def update_action(id: str, user: User, **kwargs) -> Response:
 
     ds = datastore()
 
-    existing_action = ds.action.get(id, as_obj=False)
+    existing_action, server_version = ds.action.get(id, as_obj=False, version=True)
 
     if not existing_action:
         return not_found(err="The specified automation does not exist")
@@ -166,7 +166,7 @@ def update_action(id: str, user: User, **kwargs) -> Response:
         action_obj = Action(updated_action)
         action_obj.action_id = id
 
-        ds.action.save(action_obj.action_id, action_obj, refresh=refresh)
+        ds.action.save(action_obj.action_id, action_obj, version=server_version, refresh=refresh)
     except HowlerException as e:
         return bad_request(err=str(e))
 

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/howler/api/v2/ingest.py

@@ -265,7 +265,7 @@ def overwrite(index: str, id: str, **kwargs):
 
     ds = datastore()
 
-    record = ds[index].get(id, as_obj=False, version=False)
+    record, server_version = ds[index].get(id, as_obj=False, version=True)
     if not record:
         return not_found(err="Record %s does not exist" % id)
 
@@ -292,7 +292,7 @@ def overwrite(index: str, id: str, **kwargs):
         ds[index].save(
             id,
             odm(new_record) if odm else new_record,
-            version=kwargs.get("server_version"),
+            version=server_version,
             refresh=refresh,
         )
 

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/howler/datastore/collection.py

@@ -16,6 +16,7 @@ from typing import Any, Callable, Dict, Generic, Literal, Optional, TypeVar, Uni
 import elasticsearch
 from datemath import dm
 from datemath.helpers import DateMathException
+from elastic_transport import ApiResponseMeta
 from opentelemetry import trace
 
 from howler import odm
@@ -1166,6 +1167,17 @@ class ESCollection(Generic[ModelType]):
             )
             return self._search_exists(key)
 
+    def _raise_document_not_found(self, key: str, result: Any) -> typing.NoReturn:
+        """Raise the same error shape as an Elasticsearch document lookup for an empty search."""
+        meta = ApiResponseMeta(
+            status=404,
+            http_version="1.1",
+            headers=cast(Any, {}),
+            duration=0.0,
+            node=cast(Any, None),
+        )
+        raise elasticsearch.exceptions.NotFoundError(f"Document with id {key} not found", meta, result)
+
     @overload
     def _get(self, key, retries, version: Literal[False]) -> dict[str, Any]: ...
 
@@ -1176,7 +1188,7 @@ class ESCollection(Generic[ModelType]):
         """Versioned get-save for atomic update has two paths:
             1. Document doesn't exist at all. Create token will be returned for version.
                This way only the first query to try and create the document will succeed.
-            2. Document exists in hot. A version string with the info needed to do a versioned save is returned.
+            2. Document exists. A version string with the info needed to do a versioned save is returned.
 
         The create token is needed to differentiate between "I'm saving a new
         document non-atomic (version=None)" and "I'm saving a new document
@@ -1195,11 +1207,29 @@ class ESCollection(Generic[ModelType]):
         done = False
         while not done:
             try:
-                doc = self.with_retries(self.datastore.client.get, index=self.name, id=key)
+                if self.ilm_config:
+                    result = self.with_retries(
+                        self.datastore.client.search,
+                        index=self.name,
+                        query={"ids": {"values": [key]}},
+                        size=1,
+                        seq_no_primary_term=True,
+                    )
+                    hits = result["hits"]["hits"]
+                    if not hits:
+                        self._raise_document_not_found(key, result)
+                    doc = hits[0]
+                else:
+                    doc = self.with_retries(self.datastore.client.get, index=self.name, id=key)
+
                 if version:
+                    if self.ilm_config:
+                        version_token = f"{doc['_index']}---{doc['_seq_no']}---{doc['_primary_term']}"
+                    else:
+                        version_token = f"{doc['_seq_no']}---{doc['_primary_term']}"
                     return (
                         normalize_output(doc["_source"]),
-                        f"{doc['_seq_no']}---{doc['_primary_term']}",
+                        version_token,
                     )
                 return normalize_output(doc["_source"])
             except elasticsearch.exceptions.NotFoundError:
@@ -1218,6 +1248,19 @@ class ESCollection(Generic[ModelType]):
 
         return None
 
+    def _get_version_write_target(self, version: str) -> tuple[str, str, str]:
+        """Return the concrete write target and optimistic-concurrency values from a version token."""
+        version_parts = version.split("---")
+        if len(version_parts) == 3:
+            index, seq_no, primary_term = version_parts
+            return index, seq_no, primary_term
+
+        if len(version_parts) == 2:
+            seq_no, primary_term = version_parts
+            return self.name, seq_no, primary_term
+
+        raise DataStoreException(f"Invalid version token for {self.name}: {version!r}")
+
     @overload
     def get(self, key, as_obj: Literal[True], version: Literal[True]) -> tuple[ModelType | None, str]: ...
 
@@ -1347,13 +1390,17 @@ class ESCollection(Generic[ModelType]):
 
         saved_data["id"] = key
         operation = "index"
+        index = self.name
         seq_no = None
         primary_term = None
 
+        if version is None and self.ilm_config:
+            _, version = self.get_if_exists(key, as_obj=False, version=True)
+
         if version == CREATE_TOKEN:
             operation = "create"
         elif version:
-            seq_no, primary_term = version.split("---")
+            index, seq_no, primary_term = self._get_version_write_target(version)
 
         if refresh == "true":
             logger.warning(
@@ -1363,7 +1410,7 @@ class ESCollection(Generic[ModelType]):
         try:
             self.with_retries(
                 self.datastore.client.index,
-                index=self.name,
+                index=index,
                 id=key,
                 document=json.dumps(saved_data),
                 op_type=operation,
@@ -1563,15 +1610,16 @@ class ESCollection(Generic[ModelType]):
         """
         operations = self._validate_operations(operations)
         script = self._create_scripts_from_operations(operations)
+        index = self.name
         seq_no = None
         primary_term = None
         if version:
-            seq_no, primary_term = version.split("---")
+            index, seq_no, primary_term = self._get_version_write_target(version)
 
         try:
             res = self.with_retries(
                 self.datastore.client.update,
-                index=self.name,
+                index=index,
                 id=key,
                 script=script,
                 if_seq_no=seq_no,
@@ -1581,7 +1629,11 @@ class ESCollection(Generic[ModelType]):
             )
             return (
                 res["result"] == "updated",
-                f"{res['_seq_no']}---{res['_primary_term']}",
+                (
+                    f"{res['_index']}---{res['_seq_no']}---{res['_primary_term']}"
+                    if self.ilm_config
+                    else f"{res['_seq_no']}---{res['_primary_term']}"
+                ),
             )
         except elasticsearch.NotFoundError as e:
             logger.warning("Update - elasticsearch.NotFoundError: %s %s", e.message, e.info)

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/howler/odm/mixins.py

@@ -9,7 +9,7 @@ Elasticsearch datastore without boilerplate.
 from __future__ import annotations
 
 from operator import attrgetter
-from typing import Generic, Literal, TypeVar, overload
+from typing import Generic, Literal, Self, TypeVar, overload
 
 from howler.common.exceptions import HowlerRuntimeError
 from howler.common.loader import datastore
@@ -28,10 +28,10 @@ class _ObjectsDescriptor(Generic[ModelType]):
     """
 
     @overload
-    def __get__(self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
+    def __get__(self: Self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
 
     @overload
-    def __get__(self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
+    def __get__(self: Self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
 
     def __get__(self, obj: ModelType | None, objtype: type[ModelType] | None = None) -> ESCollection[ModelType]:
         """Return the ESCollection for the owner class.
@@ -82,7 +82,11 @@ class DatastoreMixin(Generic[ModelType]):
         """
         return datastore()
 
-    def save(self, refresh: Literal["true", "false", "wait_for"] | None = None) -> bool:
+    def save(
+        self: Self,
+        refresh: Literal["true", "false", "wait_for"] | None = None,
+        version: str | None = None,
+    ) -> bool:
         """Persist the current model instance to the datastore.
 
         Determines the target index from the lowercase class name, extracts the
@@ -96,5 +100,6 @@ class DatastoreMixin(Generic[ModelType]):
         index_name = self.__class__.__name__.lower()
         id_field = self.__class__._Model__id_field  # type: ignore[attr-defined]
         current_id = attrgetter(id_field)(self)
+        collection = self.ds[index_name]
 
-        return self.ds[index_name].save(current_id, self, refresh=refresh)
+        return collection.save(current_id, self, version=version, refresh=refresh)

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/howler/services/case_service.py

@@ -19,6 +19,7 @@ from howler.common.exceptions import (
 from howler.common.loader import APP_NAME, datastore
 from howler.common.logging import get_logger
 from howler.config import CLASSIFICATION
+from howler.datastore.collection import CREATE_TOKEN
 from howler.datastore.exceptions import DataStoreException
 from howler.odm.models.case import Case, CaseItem, CaseItemTypes, CaseLog, CaseRule
 from howler.odm.models.ecs.related import Related
@@ -56,7 +57,7 @@ def create_case(
 
     case = Case(case_data)
     case.log = [CaseLog({"timestamp": "NOW", "explanation": "Case created", "user": user.uname if user else "system"})]
-    case.save(refresh="wait_for")
+    case.save(refresh="wait_for", version=CREATE_TOKEN)
     CREATED_CASES.inc()
 
     for item in items:
@@ -693,18 +694,18 @@ def remove_case_items(  # noqa: C901
 
     # Resolve backing objects for back-reference cleanup
     items_to_remove = [items_by_id[iid] for iid in ids_to_remove if iid in items_by_id]
-    backing_objs: list[Hit | Event] = []
+    backing_objs: list[tuple[Hit | Event, str]] = []
     for item in items_to_remove:
         if item.type in [CaseItemTypes.HIT, CaseItemTypes.EVENT]:
-            obj = ds[item.type].get(item.value)
+            obj, version = ds[item.type].get(item.value, as_obj=True, version=True)
             if obj:
-                backing_objs.append(obj)
+                backing_objs.append((obj, version))
 
     case.items = [item for item in case.items if item.id not in ids_to_remove]
 
-    for backing_obj in backing_objs:
+    for backing_obj, version in backing_objs:
         remove_backreference(backing_obj, case.case_id)
-        backing_obj.save()
+        backing_obj.save(version=version)
 
     recompute_case_metadata(case)
 
@@ -745,7 +746,7 @@ def append_hit(case: Case, item: CaseItem, refresh: Literal["true", "false", "wa
     """
     ds = datastore()
 
-    hit = ds.hit.get(item.value)
+    hit, version = ds.hit.get(item.value, as_obj=True, version=True)
     if hit is None:
         raise NotFoundException(f"Hit {item.value} not found, cannot be added to case")
 
@@ -754,7 +755,7 @@ def append_hit(case: Case, item: CaseItem, refresh: Literal["true", "false", "wa
     case.items.append(item)
 
     add_backreference(hit, case.case_id)
-    hit.save()
+    hit.save(version=version)
 
     recompute_case_metadata(case)
     if not case.save(refresh=refresh):  # pragma: no cover
@@ -783,7 +784,7 @@ def append_event(case: Case, item: CaseItem, refresh: Literal["true", "false", "
     """
     ds = datastore()
 
-    event = ds.event.get(key=item.value)
+    event, version = ds.event.get(key=item.value, as_obj=True, version=True)
 
     if event is None:
         raise NotFoundException(f"Event {item.value} not found, cannot be added to case")
@@ -793,7 +794,7 @@ def append_event(case: Case, item: CaseItem, refresh: Literal["true", "false", "
     case.items.append(item)
 
     add_backreference(event, case.case_id)
-    event.save()
+    event.save(version=version)
 
     recompute_case_metadata(case)
     if not case.save(refresh=refresh):  # pragma: no cover

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/howler/services/dossier_service.py

@@ -199,7 +199,7 @@ def update_dossier(  # noqa: C901
     storage = datastore()
 
     # Retrieve the existing dossier for access control checks
-    existing_dossier: Dossier = get_dossier(dossier_id, as_odm=True)
+    existing_dossier, server_version = get_dossier(dossier_id, as_odm=True, version=True)
 
     # Enforce access control for personal dossiers
     # Only the owner or admin users can modify personal dossiers
@@ -228,7 +228,7 @@ def update_dossier(  # noqa: C901
         # Merge the new data with existing dossier data
         new_data = Dossier(cast(dict, merge({}, existing_dossier.as_primitives(), dossier_data)))
 
-        storage.dossier.save(dossier_id, new_data, refresh=refresh)
+        storage.dossier.save(dossier_id, new_data, version=server_version, refresh=refresh)
 
         return new_data
     except SearchException:

{howler_api-4.0.3.dev1397 → howler_api-4.0.4.dev1401}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.3.dev1397"
+version = "4.0.4.dev1401"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",