howler-api 3.0.0.dev363__tar.gz → 3.0.0.dev366__tar.gz

{howler_api-3.0.0.dev363 → howler_api-3.0.0.dev366}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 3.0.0.dev363
+Version: 3.0.0.dev366
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-3.0.0.dev363 → howler_api-3.0.0.dev366}/howler/datastore/collection.py

@@ -418,8 +418,11 @@ class ESCollection(Generic[ModelType]):
         #   - update_by_query
         #   - delete_by_query
         #   - reindex ...
+        attempt = 0
         res = None
         while res is None:
+            attempt = attempt + 1
+            logger.warning("Checking status of task %s (Attempt %s)", task["task"], attempt)
             try:
                 res = self.with_retries(
                     self.datastore.client.tasks.get,
@@ -435,9 +438,14 @@ class ESCollection(Generic[ModelType]):
                 ]:
                     pass
                 else:
+                    logger.exception("Unexpected error on task check")
                     raise
 
-        return res.get("response", res["task"]["status"])
+        result = res.get("response", res["task"]["status"])
+
+        logger.info("Task result:\n%s", str(result))
+
+        return result
 
     def _get_current_alias(self, index: str) -> typing.Optional[str]:
         if self.with_retries(self.datastore.client.indices.exists_alias, name=index):
@@ -679,6 +687,7 @@ class ESCollection(Generic[ModelType]):
 
         :return: Should return True of the commit was successful on all hosts
         """
+        logger.warning("Beginning Reindex")
         for index in self.index_list:
             new_name = f"{index}__reindex"
             index_data = None
@@ -689,6 +698,7 @@ class ESCollection(Generic[ModelType]):
                 index_data = self.with_retries(self.datastore.client.indices.get, index=index)[index]
 
                 # Create reindex target
+                logger.warning("Creating new index with name %s", new_name)
                 self.with_retries(
                     self.datastore.client.indices.create,
                     index=new_name,
@@ -719,36 +729,39 @@ class ESCollection(Generic[ModelType]):
                     else:
                         alias_actions = [{"add": {"index": new_name, "alias": alias}}]
 
+                    logger.warning("Updating alias %s", alias)
                     self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
 
                 # Reindex data into target
+                logger.warning("Beginning reindex from %s to %s", index, new_name)
                 r_task = self.with_retries(
                     self.datastore.client.reindex,
                     source={"index": index},
                     dest={"index": new_name},
                     wait_for_completion=False,
                 )
+                logger.warning("Reindex taskId: %s", r_task["task"])
                 self._get_task_results(r_task)
 
             if self.with_retries(self.datastore.client.indices.exists, index=new_name):
                 if index_data is None:
                     index_data = self.with_retries(self.datastore.client.indices.get, index=index)[index]
 
-                # Commit reindexed data
+                logger.warning("Committing reindexed data in index %s", new_name)
                 self.with_retries(self.datastore.client.indices.refresh, index=new_name)
                 self.with_retries(self.datastore.client.indices.clear_cache, index=new_name)
 
-                # Delete old index
+                logger.warning("Deleting old index %s", index)
                 if self.with_retries(self.datastore.client.indices.exists, index=index):
                     self.with_retries(self.datastore.client.indices.delete, index=index)
 
-                # Block write to the index
+                logger.warning("Block write to index")
                 self.with_retries(
                     self.datastore.client.indices.put_settings,
                     settings=write_block_settings,
                 )
 
-                # Rename reindexed index
+                logger.warning("Renaming reindexed index from %s to %s", new_name, index)
                 try:
                     self._safe_index_copy(
                         self.datastore.client.indices.clone,
@@ -776,11 +789,11 @@ class ESCollection(Generic[ModelType]):
                                 actions=alias_actions,
                             )
 
-                    # Delete the reindex target if it still exists
                     if self.with_retries(self.datastore.client.indices.exists, index=new_name):
+                        logger.warning("Deleting reindex target %s", new_name)
                         self.with_retries(self.datastore.client.indices.delete, index=new_name)
                 finally:
-                    # Unblock write to the index
+                    logger.warning("Unblock write to the index")
                     self.with_retries(
                         self.datastore.client.indices.put_settings,
                         settings=write_unblock_settings,

{howler_api-3.0.0.dev363 → howler_api-3.0.0.dev366}/howler/external/reindex_data.py

@@ -1,3 +1,4 @@
+import json
 import sys
 import time
 from typing import Callable
@@ -45,7 +46,10 @@ if __name__ == "__main__":
         print("Invalid index provided, stopping.")
         sys.exit(1)
 
-    print("You will be reindexing the following indexes:")
+    print("Index schema:")
+    print(json.dumps(ds.hit._get_index_mappings(), indent=2))
+
+    print("\nYou will be reindexing the following indexes:")
     print("\n".join(indexes[index_answer][0]))
 
     answer = input(("\nAre you sure you want to reindex these indexes? [yes/NO]\n"))

{howler_api-3.0.0.dev363 → howler_api-3.0.0.dev366}/pyproject.toml

@@ -148,7 +148,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "3.0.0.dev363"
+version = "3.0.0.dev366"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",