howler-api 4.0.0.dev1308__tar.gz → 4.0.0.dev1337__tar.gz

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1308
+Version: 4.0.0.dev1337
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -38,7 +38,7 @@ Requires-Dist: opentelemetry-instrumentation-flask (==0.61b0)
 Requires-Dist: opentelemetry-sdk (==1.40.0)
 Requires-Dist: packaging (<25.0)
 Requires-Dist: passlib (==1.7.4)
-Requires-Dist: prometheus-client (==0.25.0)
+Requires-Dist: prometheus-client (==0.26.0)
 Requires-Dist: pydantic (>=2.11.4,<3.0.0)
 Requires-Dist: pydantic-settings[yaml] (>=2.9.1,<3.0.0)
 Requires-Dist: pydash (>=8.0.5,<9.0.0)

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/howler/datastore/README.md

@@ -37,10 +37,9 @@ from howler.common import loader
 from myapp.models.mymodel import MyModel
 
 ds = loader.get_esstore()
-ds.register('mymodel', MyModel)
+ds.register("mymodel", MyModel)
 
 my_document = ds.mymodel.get(document_id)
-
 ```
 
 ### Creating your own datastore
@@ -59,10 +58,10 @@ from myapp.models.mycollection import MyCollection
 
 
 class MyDatastore(object):
-    def __init__(self, esstore_object: ESStore = None ):
+    def __init__(self, esstore_object: ESStore = None):
 
         self.ds = esstore_object or loader.get_esstore()
-        self.ds.register('mycollection', MyCollection)
+        self.ds.register("mycollection", MyCollection)
         # ... + all other collections
 
     def __enter__(self):

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/howler/datastore/collection.py

@@ -252,19 +252,50 @@ class ESCollection(Generic[ModelType]):
 
     @property
     def index_list_full(self):
+        """Return every physical index for this collection, newest ILM index first.
+
+        Maintenance operations that must visit historical rollover indexes should use this
+        property. The active index is included even when no ILM rollover index exists.
+        """
         if not self._index_list:
-            self._index_list = list(self.with_retries(self.datastore.client.indices.get, index=f"{self.name}-*").keys())
+            self._index_list = self._get_ilm_index_names()
 
-        return [self.index_name] + sorted(self._index_list, reverse=True)
+        return list(dict.fromkeys([self.index_name, *sorted(self._index_list, reverse=True)]))
 
     @property
     def index_list(self):
-        """This property contains the list of valid indexes for the current collection.
+        """Return the active physical write index for this collection.
 
-        :return: list of valid indexes for this collection
+        Historical ILM rollover indexes are intentionally excluded. Bulk plans use this
+        list for default write, update, and delete targets, and widening it would fan those
+        operations out across historical indexes.
         """
         return [self.index_name]
 
+    def _get_ilm_index_names(self) -> list[str]:
+        """Return physical ILM rollover indexes, excluding temporary maintenance indexes."""
+        try:
+            indices = self.datastore.client.indices.get(
+                index=f"{self.name}-0*",
+                ignore_unavailable=True,
+            )
+        except elasticsearch.exceptions.NotFoundError:
+            return []
+
+        index_pattern = re.compile(rf"^{re.escape(self.name)}-\d{{6}}$")
+        return sorted(index for index in indices if index_pattern.fullmatch(index))
+
+    def _refresh_ilm_index_name(self):
+        """Set ``index_name`` to the latest existing ILM index, when one exists."""
+        if not self.ilm_config:
+            return
+
+        ilm_indices = self._get_ilm_index_names()
+
+        if ilm_indices:
+            self._index_list = ilm_indices
+            self.index_name = ilm_indices[-1]
+
     def scan_with_retry(
         self,
         query,
@@ -730,7 +761,7 @@ class ESCollection(Generic[ModelType]):
     def reindex(self, allow_failures: bool = False, request_timeout: int = 60):
         """Reindex all the data of the collection into a freshly mapped index.
 
-        For every index in ``self.index_list`` the data is copied into a temporary
+        For every physical index in ``self.index_list_full`` the data is copied into a temporary
         ``__reindex`` index that uses the current mappings/settings. Writes to the source
         index are blocked while the copy runs so the reindex result and document counts can
         be validated before the original index is deleted. The temporary index is only
@@ -750,7 +781,8 @@ class ESCollection(Generic[ModelType]):
             operation can be recovered with :meth:`reindex_cleanup`.
         """
         logger.warning("Beginning Reindex")
-        for index in self.index_list:
+        self._refresh_ilm_index_name()
+        for index in self.index_list_full:
             new_name = f"{index}__reindex"
             index_data = None
             source_count = None
@@ -798,7 +830,7 @@ class ESCollection(Generic[ModelType]):
                     self.datastore.client.indices.create,
                     index=new_name,
                     mappings=self._get_index_mappings(),
-                    settings=self._get_index_settings(),
+                    settings=self._get_reindex_settings(index_data),
                 )
 
                 # Reindex data into target
@@ -847,12 +879,10 @@ class ESCollection(Generic[ModelType]):
                     )
 
                     alias_actions = []
-                    aliases = index_data.get("aliases", {}) or {self.name: {"is_write_index": True}}
+                    aliases = index_data.get("aliases", {})
                     for alias, alias_data in aliases.items():
                         alias_action = {"index": index, "alias": alias}
                         alias_action.update(alias_data)
-                        if alias == self.name or alias_data.get("is_write_index", False):
-                            alias_action["is_write_index"] = True
                         alias_actions.append({"add": alias_action})
                     alias_actions.append({"remove_index": {"index": new_name}})
                     self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
@@ -880,6 +910,14 @@ class ESCollection(Generic[ModelType]):
 
         return True
 
+    def _get_reindex_settings(self, index_data: dict) -> dict:
+        """Build settings for a reindex target without dropping ILM lifecycle metadata."""
+        settings = self._get_index_settings()
+        lifecycle = index_data.get("settings", {}).get("index", {}).get("lifecycle")
+        if lifecycle:
+            settings["index"]["lifecycle"] = deepcopy(lifecycle)
+        return settings
+
     def _validate_reindex_result(self, index, new_name, reindex_result, allow_failures):
         """Validate the result of an Elasticsearch reindex task.
 
@@ -956,17 +994,18 @@ class ESCollection(Generic[ModelType]):
     def reindex_cleanup(self):
         """Recover from a failed or interrupted :meth:`reindex` run.
 
-        For every index that still has a leftover ``__reindex`` index, restore the source
-        index as the writable target and delete the orphaned ``__reindex`` index. If the
-        source index is missing, raise an error instead of deleting ``__reindex`` because
-        it may be the only remaining copy of the data.
+        For every physical index that still has a leftover ``__reindex`` index, restore the source
+        index's original aliases and delete the orphaned ``__reindex`` index. If the source
+        index is missing, raise an error instead of deleting ``__reindex`` because it may
+        be the only remaining copy of the data.
 
         Write blocks set during reindexing are cleared so the collection is usable again.
 
         :return: ``True`` when cleanup completed on all indexes
         """
         logger.warning("Beginning reindex cleanup")
-        for index in self.index_list:
+        self._refresh_ilm_index_name()
+        for index in self.index_list_full:
             new_name = f"{index}__reindex"
 
             if not self.with_retries(self.datastore.client.indices.exists, index=new_name):
@@ -987,17 +1026,13 @@ class ESCollection(Generic[ModelType]):
             reindex_aliases = new_index_data.get("aliases", {})
 
             alias_actions = []
-            for alias in sorted(set(source_aliases) | set(reindex_aliases) | {self.name}):
+            for alias in sorted(set(source_aliases) | set(reindex_aliases)):
                 source_alias_data = source_aliases.get(alias, {})
                 reindex_alias_data = reindex_aliases.get(alias, {})
 
                 add_alias_data = {"index": index, "alias": alias}
                 add_alias_data.update(source_alias_data)
-                if (
-                    alias == self.name
-                    or source_alias_data.get("is_write_index", False)
-                    or reindex_alias_data.get("is_write_index", False)
-                ):
+                if source_alias_data.get("is_write_index", False) or reindex_alias_data.get("is_write_index", False):
                     add_alias_data["is_write_index"] = True
 
                 alias_actions.append({"add": add_alias_data})
@@ -2905,7 +2940,42 @@ class ESCollection(Generic[ModelType]):
         if existing_ilm_indices:
             # ILM already bootstrapped — ensure the alias exists
             latest = sorted(existing_ilm_indices)[-1]
-            if not self.with_retries(self.datastore.client.indices.exists_alias, name=self.name):
+            legacy_hot_index = f"{self.name}_hot"
+            if self.with_retries(self.datastore.client.indices.exists_alias, name=self.name):
+                alias_indices = self.with_retries(self.datastore.client.indices.get_alias, name=self.name)
+                alias_actions = []
+                for alias_index, alias_index_data in alias_indices.items():
+                    alias_data = alias_index_data.get("aliases", {}).get(self.name, {})
+                    if alias_index == legacy_hot_index:
+                        alias_actions.append({"remove": {"index": alias_index, "alias": self.name}})
+                    elif alias_index != latest and alias_data.get("is_write_index", False):
+                        alias_actions.append(
+                            {
+                                "add": {
+                                    "index": alias_index,
+                                    "alias": self.name,
+                                    **alias_data,
+                                    "is_write_index": False,
+                                }
+                            }
+                        )
+
+                latest_alias_data = alias_indices.get(latest, {}).get("aliases", {}).get(self.name, {})
+                if not latest_alias_data.get("is_write_index", False):
+                    alias_actions.append(
+                        {
+                            "add": {
+                                "index": latest,
+                                "alias": self.name,
+                                **latest_alias_data,
+                                "is_write_index": True,
+                            }
+                        }
+                    )
+
+                if alias_actions:
+                    self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
+            else:
                 # Find the latest index to set as write index
                 self.with_retries(
                     self.datastore.client.indices.put_alias,

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/howler/external/reindex_data.py

@@ -99,6 +99,7 @@ if __name__ == "__main__":
 
     for index_name in selected:
         collection: ESCollection = getattr(ds, index_name)
+        collection._refresh_ilm_index_name()
 
         if args.verbose:
             print(f"Index schema for '{index_name}':")

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/howler/odm/README.md

@@ -11,15 +11,18 @@ Here's example of a user model with settings and stats:
 ```python
 from howler import odm
 
+
 @odm.model(index=True, store=False, description="Settings of user")
 class Settings(odm.Model):
-    default_view: str = odm.Enum(value=['detail', 'simple'], default='simple', description="Some random setting")
+    default_view: str = odm.Enum(value=["detail", "simple"], default="simple", description="Some random setting")
+
 
 @odm.model(index=True, store=False, description="Settings of user")
 class Stats(odm.Model):
-    last_login: str = odm.Date(default='NOW', description="Last time user logged in")
+    last_login: str = odm.Date(default="NOW", description="Last time user logged in")
     login_count: str = odm.Integer(default=0, description="Number of time the user logged in")
 
+
 @odm.model(index=True, store=True, description="User example")
 class User(odm.Model):
     username: str = odm.Keyword(description="Username of the user")

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/howler/odm/models/case.py

@@ -153,17 +153,17 @@ class CaseTask(odm.Model):
     item: str = odm.Keyword(description="Associated case item id.", optional=True)
 
 
-@odm.model(index=True, store=True, description="Enrichment annotations associated with a case path.")
+@odm.model(index=True, store=True, description="Enrichment annotations associated with a case item id.")
 class CaseEnrichment(odm.Model):
-    path: str = odm.Keyword(description="Case item path associated with these annotations.")
+    item: str = odm.Keyword(description="Case item id associated with these annotations.")
     annotations: list[str] = odm.List(
         odm.Keyword(),
         default=[],
-        description="Annotation IDs associated with the path.",
+        description="Annotation IDs associated with the case item id.",
     )
 
 
-@odm.model(index=True, store=True, description="Case model with path-based items, enrichments, rules, and tasks.")
+@odm.model(index=True, store=True, description="Case model with items, enrichments, rules, and tasks.")
 class Case(DatastoreMixin["Case"], odm.Model):
     case_id: str = odm.UUID(description="A unique identifier for this case.")
     classification: str = odm.Classification(

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/howler/odm/random_data.py

@@ -867,8 +867,12 @@ def wipe_actions(ds: HowlerDatastore):
 def create_dossiers(ds: HowlerDatastore, num_dossiers: int = 5):
     "Create random dossiers"
     users = ds.user.search("*:*")["items"]
-    for _ in range(num_dossiers):
+    for index in range(num_dossiers):
         dossier = generate_useful_dossier(users)
+        if index == 0:
+            dossier.type = "personal"
+        elif index == 1:
+            dossier.type = "global"
         ds.dossier.save(dossier.dossier_id, dossier)
 
     ds.dossier.commit()

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/howler/remote/datatypes/README.md

@@ -25,9 +25,10 @@ Example:
 
 ```python
 from howler.remote.datatypes.counters import Counters
-with Counters('test-counter') as ct:
+
+with Counters("test-counter") as ct:
     # Increment counter by 4
-    ct.inc('value_1', 4)
+    ct.inc("value_1", 4)
 
     # Get Counter values
     ct.get_queues_sizes()
@@ -60,19 +61,20 @@ Example, A container register a watcher for a specific event:
 import time
 from howler.remote.datatypes.events import EventWatcher
 
+
 def callback_func(data: dict[str, Any]):
     print(data)
 
+
 watcher = EventWatcher()
 try:
     # register for the test event in the event group
-    watcher.register('event.test', callback_func)
+    watcher.register("event.test", callback_func)
     watcher.start()
     while True:
         time.sleep(1)
 finally:
     watcher.stop()
-
 ```
 
 Another container creates an event to wake up the first container:
@@ -82,10 +84,10 @@ import time
 from howler.remote.datatypes.events import EventSender
 
 # create a sender for the event group
-sender = EventSender('event.', redis_connection)
+sender = EventSender("event.", redis_connection)
 
 # send a test even
-sender.send('test', {'payload': 100})
+sender.send("test", {"payload": 100})
 # After this call, the first container will have woken up and executed its callback function
 # that would print: {'payload': 100}
 ```
@@ -114,7 +116,8 @@ Example:
 
 ```python
 from howler.remote.datatypes.hash import Hash
-with Hash('test-hashmap') as h:
+
+with Hash("test-hashmap") as h:
     h.add("key", "value")
     print(h.exists("key"))
     print(h.get("key"))
@@ -172,10 +175,11 @@ Example:
 
 ```python
 from howler.remote.datatypes.set import Set
-with Set('test-set') as s:
-    values = ['a', 'b', 1, 2]
+
+with Set("test-set") as s:
+    values = ["a", "b", 1, 2]
     s.add(*values)
-    print(s.exist('b'))
+    print(s.exist("b"))
     print(s.pop())
     print(s.length())
 ```
@@ -184,7 +188,7 @@ Output:
 
 ```python
 True
-'a' or 'b' or 1 or 2
+"a" or "b" or 1 or 2
 3
 ```
 
@@ -234,7 +238,7 @@ Example of message receiver:
 ```python
 from howler.remote.datatypes.queues.comms import CommsQueue
 
-with CommsQueue('test-comms-queue') as cq:
+with CommsQueue("test-comms-queue") as cq:
     for msg in cq.listen(blocking=True):
         if msg == "stop":
             break
@@ -247,7 +251,7 @@ Example of message sender:
 ```python
 from howler.remote.datatypes.queues.comms import CommsQueue
 
-with CommsQueue('test-comms-queue') as cq:
+with CommsQueue("test-comms-queue") as cq:
     # send a message to the receiver
     cq.publish("This is my message!")
 
@@ -273,7 +277,8 @@ Example:
 
 ```python
 from howler.remote.datatypes.queues.named import NamedQueue, select
-with NamedQueue('test-named-queue') as nq:
+
+with NamedQueue("test-named-queue") as nq:
     for x in range(5):
         nq.push(x)
 
@@ -307,12 +312,13 @@ Example:
 
 ```python
 from howler.remote.datatypes.queues.priority import PriorityQueue, length, select
-with PriorityQueue('test-priority-queue') as pq:
+
+with PriorityQueue("test-priority-queue") as pq:
     for x in range(10):
         pq.push(100, x)
 
-    a_key = pq.push(101, 'a')
-    z_key = pq.push(99, 'z')
+    a_key = pq.push(101, "a")
+    z_key = pq.push(99, "z")
     print(pq.rank(a_key))
     print(pq.rank(z_key))
     print(pq.pop())
@@ -323,7 +329,7 @@ Output:
 ```python
 0
 11
-'a'
+"a"
 ```
 
 #### Unique Priority queues
@@ -334,7 +340,8 @@ Example:
 
 ```python
 from howler.remote.datatypes.queues.priority import PriorityQueue, length, select
-with PriorityQueue('test-priority-queue') as pq:
+
+with PriorityQueue("test-priority-queue") as pq:
     for x in range(10):
         pq.push(100, x)
 

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1337}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev1308"
+version = "4.0.0.dev1337"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",
@@ -206,7 +206,7 @@ gevent = "^25.9.1"
 flask-caching = "2.4.1"
 packaging = "<25.0"
 passlib = "1.7.4"
-prometheus-client = "0.25.0"
+prometheus-client = "0.26.0"
 pyjwt = "2.13.0"
 python-baseconv = "1.2.2"
 python-datemath = "3.0.3"
@@ -235,7 +235,7 @@ azure-monitor-opentelemetry = "^1.8.7"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.7.0"
-ruff = ">=0.8,<0.16"
+ruff = ">=0.8,<0.17"
 pyright = { extras = ["nodejs"], version = "^1.1.408" }
 mypy = "^1.6.1"
 
@@ -252,10 +252,10 @@ coverage = { extras = ["toml"], version = "^7.4.4" }
 
 [tool.poetry.group.types.dependencies]
 typing-extensions = "^4.12.2"
-types-PyYAML = "6.0.12.20260518"
+types-PyYAML = "6.0.12.20260724"
 types-paramiko = "3.5.0.20250801"
 types-pyOpenSSL = "23.3.0.20240106"
-types-six = "1.17.0.20260721"
+types-six = "1.17.0.20260724"
 types-requests = "<2.33.0.20260713"
 websocket-client = "1.9.0"
 types-pytz = "^2024.1.0.20240417"