howler-api 4.0.0.dev1308__tar.gz → 4.0.0.dev1311__tar.gz

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1311}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1308
+Version: 4.0.0.dev1311
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1311}/howler/datastore/collection.py

@@ -252,19 +252,50 @@ class ESCollection(Generic[ModelType]):
 
     @property
     def index_list_full(self):
+        """Return every physical index for this collection, newest ILM index first.
+
+        Maintenance operations that must visit historical rollover indexes should use this
+        property. The active index is included even when no ILM rollover index exists.
+        """
         if not self._index_list:
-            self._index_list = list(self.with_retries(self.datastore.client.indices.get, index=f"{self.name}-*").keys())
+            self._index_list = self._get_ilm_index_names()
 
-        return [self.index_name] + sorted(self._index_list, reverse=True)
+        return list(dict.fromkeys([self.index_name, *sorted(self._index_list, reverse=True)]))
 
     @property
     def index_list(self):
-        """This property contains the list of valid indexes for the current collection.
+        """Return the active physical write index for this collection.
 
-        :return: list of valid indexes for this collection
+        Historical ILM rollover indexes are intentionally excluded. Bulk plans use this
+        list for default write, update, and delete targets, and widening it would fan those
+        operations out across historical indexes.
         """
         return [self.index_name]
 
+    def _get_ilm_index_names(self) -> list[str]:
+        """Return physical ILM rollover indexes, excluding temporary maintenance indexes."""
+        try:
+            indices = self.datastore.client.indices.get(
+                index=f"{self.name}-0*",
+                ignore_unavailable=True,
+            )
+        except elasticsearch.exceptions.NotFoundError:
+            return []
+
+        index_pattern = re.compile(rf"^{re.escape(self.name)}-\d{{6}}$")
+        return sorted(index for index in indices if index_pattern.fullmatch(index))
+
+    def _refresh_ilm_index_name(self):
+        """Set ``index_name`` to the latest existing ILM index, when one exists."""
+        if not self.ilm_config:
+            return
+
+        ilm_indices = self._get_ilm_index_names()
+
+        if ilm_indices:
+            self._index_list = ilm_indices
+            self.index_name = ilm_indices[-1]
+
     def scan_with_retry(
         self,
         query,
@@ -730,7 +761,7 @@ class ESCollection(Generic[ModelType]):
     def reindex(self, allow_failures: bool = False, request_timeout: int = 60):
         """Reindex all the data of the collection into a freshly mapped index.
 
-        For every index in ``self.index_list`` the data is copied into a temporary
+        For every physical index in ``self.index_list_full`` the data is copied into a temporary
         ``__reindex`` index that uses the current mappings/settings. Writes to the source
         index are blocked while the copy runs so the reindex result and document counts can
         be validated before the original index is deleted. The temporary index is only
@@ -750,7 +781,8 @@ class ESCollection(Generic[ModelType]):
             operation can be recovered with :meth:`reindex_cleanup`.
         """
         logger.warning("Beginning Reindex")
-        for index in self.index_list:
+        self._refresh_ilm_index_name()
+        for index in self.index_list_full:
             new_name = f"{index}__reindex"
             index_data = None
             source_count = None
@@ -798,7 +830,7 @@ class ESCollection(Generic[ModelType]):
                     self.datastore.client.indices.create,
                     index=new_name,
                     mappings=self._get_index_mappings(),
-                    settings=self._get_index_settings(),
+                    settings=self._get_reindex_settings(index_data),
                 )
 
                 # Reindex data into target
@@ -847,12 +879,10 @@ class ESCollection(Generic[ModelType]):
                     )
 
                     alias_actions = []
-                    aliases = index_data.get("aliases", {}) or {self.name: {"is_write_index": True}}
+                    aliases = index_data.get("aliases", {})
                     for alias, alias_data in aliases.items():
                         alias_action = {"index": index, "alias": alias}
                         alias_action.update(alias_data)
-                        if alias == self.name or alias_data.get("is_write_index", False):
-                            alias_action["is_write_index"] = True
                         alias_actions.append({"add": alias_action})
                     alias_actions.append({"remove_index": {"index": new_name}})
                     self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
@@ -880,6 +910,14 @@ class ESCollection(Generic[ModelType]):
 
         return True
 
+    def _get_reindex_settings(self, index_data: dict) -> dict:
+        """Build settings for a reindex target without dropping ILM lifecycle metadata."""
+        settings = self._get_index_settings()
+        lifecycle = index_data.get("settings", {}).get("index", {}).get("lifecycle")
+        if lifecycle:
+            settings["index"]["lifecycle"] = deepcopy(lifecycle)
+        return settings
+
     def _validate_reindex_result(self, index, new_name, reindex_result, allow_failures):
         """Validate the result of an Elasticsearch reindex task.
 
@@ -956,17 +994,18 @@ class ESCollection(Generic[ModelType]):
     def reindex_cleanup(self):
         """Recover from a failed or interrupted :meth:`reindex` run.
 
-        For every index that still has a leftover ``__reindex`` index, restore the source
-        index as the writable target and delete the orphaned ``__reindex`` index. If the
-        source index is missing, raise an error instead of deleting ``__reindex`` because
-        it may be the only remaining copy of the data.
+        For every physical index that still has a leftover ``__reindex`` index, restore the source
+        index's original aliases and delete the orphaned ``__reindex`` index. If the source
+        index is missing, raise an error instead of deleting ``__reindex`` because it may
+        be the only remaining copy of the data.
 
         Write blocks set during reindexing are cleared so the collection is usable again.
 
         :return: ``True`` when cleanup completed on all indexes
         """
         logger.warning("Beginning reindex cleanup")
-        for index in self.index_list:
+        self._refresh_ilm_index_name()
+        for index in self.index_list_full:
             new_name = f"{index}__reindex"
 
             if not self.with_retries(self.datastore.client.indices.exists, index=new_name):
@@ -987,17 +1026,13 @@ class ESCollection(Generic[ModelType]):
             reindex_aliases = new_index_data.get("aliases", {})
 
             alias_actions = []
-            for alias in sorted(set(source_aliases) | set(reindex_aliases) | {self.name}):
+            for alias in sorted(set(source_aliases) | set(reindex_aliases)):
                 source_alias_data = source_aliases.get(alias, {})
                 reindex_alias_data = reindex_aliases.get(alias, {})
 
                 add_alias_data = {"index": index, "alias": alias}
                 add_alias_data.update(source_alias_data)
-                if (
-                    alias == self.name
-                    or source_alias_data.get("is_write_index", False)
-                    or reindex_alias_data.get("is_write_index", False)
-                ):
+                if source_alias_data.get("is_write_index", False) or reindex_alias_data.get("is_write_index", False):
                     add_alias_data["is_write_index"] = True
 
                 alias_actions.append({"add": add_alias_data})
@@ -2905,7 +2940,42 @@ class ESCollection(Generic[ModelType]):
         if existing_ilm_indices:
             # ILM already bootstrapped — ensure the alias exists
             latest = sorted(existing_ilm_indices)[-1]
-            if not self.with_retries(self.datastore.client.indices.exists_alias, name=self.name):
+            legacy_hot_index = f"{self.name}_hot"
+            if self.with_retries(self.datastore.client.indices.exists_alias, name=self.name):
+                alias_indices = self.with_retries(self.datastore.client.indices.get_alias, name=self.name)
+                alias_actions = []
+                for alias_index, alias_index_data in alias_indices.items():
+                    alias_data = alias_index_data.get("aliases", {}).get(self.name, {})
+                    if alias_index == legacy_hot_index:
+                        alias_actions.append({"remove": {"index": alias_index, "alias": self.name}})
+                    elif alias_index != latest and alias_data.get("is_write_index", False):
+                        alias_actions.append(
+                            {
+                                "add": {
+                                    "index": alias_index,
+                                    "alias": self.name,
+                                    **alias_data,
+                                    "is_write_index": False,
+                                }
+                            }
+                        )
+
+                latest_alias_data = alias_indices.get(latest, {}).get("aliases", {}).get(self.name, {})
+                if not latest_alias_data.get("is_write_index", False):
+                    alias_actions.append(
+                        {
+                            "add": {
+                                "index": latest,
+                                "alias": self.name,
+                                **latest_alias_data,
+                                "is_write_index": True,
+                            }
+                        }
+                    )
+
+                if alias_actions:
+                    self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
+            else:
                 # Find the latest index to set as write index
                 self.with_retries(
                     self.datastore.client.indices.put_alias,

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1311}/howler/external/reindex_data.py

@@ -99,6 +99,7 @@ if __name__ == "__main__":
 
     for index_name in selected:
         collection: ESCollection = getattr(ds, index_name)
+        collection._refresh_ilm_index_name()
 
         if args.verbose:
             print(f"Index schema for '{index_name}':")

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1311}/howler/odm/random_data.py

@@ -867,8 +867,12 @@ def wipe_actions(ds: HowlerDatastore):
 def create_dossiers(ds: HowlerDatastore, num_dossiers: int = 5):
     "Create random dossiers"
     users = ds.user.search("*:*")["items"]
-    for _ in range(num_dossiers):
+    for index in range(num_dossiers):
         dossier = generate_useful_dossier(users)
+        if index == 0:
+            dossier.type = "personal"
+        elif index == 1:
+            dossier.type = "global"
         ds.dossier.save(dossier.dossier_id, dossier)
 
     ds.dossier.commit()

{howler_api-4.0.0.dev1308 → howler_api-4.0.0.dev1311}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev1308"
+version = "4.0.0.dev1311"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",