howler-api 3.4.0.dev912__tar.gz → 3.4.0.dev927__tar.gz

{howler_api-3.4.0.dev912 → howler_api-3.4.0.dev927}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 3.4.0.dev912
+Version: 3.4.0.dev927
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-3.4.0.dev912 → howler_api-3.4.0.dev927}/howler/api/v1/search.py

@@ -78,7 +78,6 @@ def search(index, **kwargs):
     sort                =>   How to sort the results (not available in deep paging)
     fl                  =>   List of fields to return
     timeout             =>   Maximum execution time (ms)
-    use_archive         =>   Allow access to the datastore achive (Default: False)
     track_total_hits    =>   Track the total number of query matches, instead of stopping at 10000 (Default: False)
     metadata            =>   A list of additional features to be added to the result alongside the raw results
 
@@ -118,18 +117,9 @@ def search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters", "metadata"]
-    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
-    params.update(
-        {
-            k: str(req_data.get(k, "false")).lower() in ["true", ""]
-            for k in boolean_fields
-            if req_data.get(k, None) is not None
-        }
-    )
-
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -350,18 +340,9 @@ def sigma_search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters"]
-    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
-    params.update(
-        {
-            k: str(req_data.get(k, "false")).lower() in ["true", ""]
-            for k in boolean_fields
-            if req_data.get(k, None) is not None
-        }
-    )
-
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -520,7 +501,6 @@ def count(index, **kwargs):
     Optional Arguments:
     filters             =>   List of additional filter queries limit the data
     timeout             =>   Maximum execution time (ms)
-    use_archive         =>   Allow access to the datastore achive (Default: False)
 
     Data Block:
     # Note that the data block is for POST requests only!
@@ -543,15 +523,6 @@ def count(index, **kwargs):
 
     params, req_data = generate_params(request, [], [])
 
-    boolean_fields = ["use_archive"]
-    params.update(
-        {
-            k: str(req_data.get(k, "false")).lower() in ["true", ""]
-            for k in boolean_fields
-            if req_data.get(k, None) is not None
-        }
-    )
-
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 

{howler_api-3.4.0.dev912 → howler_api-3.4.0.dev927}/howler/datastore/collection.py

@@ -215,7 +215,7 @@ class ESCollection(Generic[ModelType]):
     IGNORE_ENSURE_COLLECTION: bool = False
     ENSURE_COLLECTION_WARNED: bool = False
 
-    def __init__(self, datastore: ESStore, name, model_class=None, validate=True, max_attempts=10):
+    def __init__(self, datastore: ESStore, name, model_class=None, validate=True, max_attempts=10, ilm_config=None):
         self.replicas = int(
             environ.get(
                 f"ELASTIC_{name.upper()}_REPLICAS",
@@ -227,6 +227,7 @@ class ESCollection(Generic[ModelType]):
 
         self.datastore = datastore
         self.name = f"{APP_NAME}-{name}"
+        self.ilm_config = ilm_config
         self.index_name = f"{self.name}_hot"
         self.model_class = model_class
         self.validate = validate
@@ -1394,7 +1395,7 @@ class ESCollection(Generic[ModelType]):
 
         return prune(source_data, fields, self.stored_fields, mapping_class=Mapping)
 
-    def _search(self, args=None, deep_paging_id=None, use_archive=False, track_total_hits=None):
+    def _search(self, args=None, deep_paging_id=None, track_total_hits=None):
         if args is None:
             args = []
 
@@ -1551,7 +1552,6 @@ class ESCollection(Generic[ModelType]):
         filters: list[str] | str | None = None,
         access_control: typing.Any = None,
         deep_paging_id: str | None = None,
-        use_archive: bool = False,
         track_total_hits: bool = False,
         script_fields: list[str] = [],
     ) -> SearchResult[ModelType]: ...
@@ -1569,7 +1569,6 @@ class ESCollection(Generic[ModelType]):
         filters: list[str] | str | None = None,
         access_control: typing.Any = None,
         deep_paging_id: str | None = None,
-        use_archive: bool = False,
         track_total_hits: bool = False,
         script_fields: list[str] = [],
     ) -> SearchResult[dict[str, typing.Any]]: ...
@@ -1586,7 +1585,6 @@ class ESCollection(Generic[ModelType]):
         filters=None,
         access_control=None,
         deep_paging_id=None,
-        use_archive=False,
         track_total_hits=None,
         script_fields=[],
     ):
@@ -1607,7 +1605,6 @@ class ESCollection(Generic[ModelType]):
 
         :param script_fields: List of name/script tuple of fields to be evaluated at runtime
         :param track_total_hits: Return to total matching document count
-        :param use_archive: Query also the archive
         :param deep_paging_id: ID of the next page during deep paging searches
         :param as_obj: Return objects instead of dictionaries
         :param query: lucene query to search for
@@ -1663,7 +1660,6 @@ class ESCollection(Generic[ModelType]):
         result = self._search(
             args,
             deep_paging_id=deep_paging_id,
-            use_archive=use_archive,
             track_total_hits=track_total_hits,
         )
 
@@ -1710,7 +1706,6 @@ class ESCollection(Generic[ModelType]):
         access_control=None,
         item_buffer_size=200,
         as_obj=True,
-        use_archive=False,
     ):
         """This function should perform a search through the datastore and stream
         all related results as a dictionary of key value pair where each keys
@@ -1723,7 +1718,6 @@ class ESCollection(Generic[ModelType]):
         >>>     fl[x]: value
         >>> }
 
-        :param use_archive: Query also the archive
         :param as_obj: Return objects instead of dictionaries
         :param query: lucene query to search for
         :param fl: list of fields to return from the search
@@ -1947,7 +1941,6 @@ class ESCollection(Generic[ModelType]):
         mincount=None,
         filters=None,
         access_control=None,
-        use_archive=False,
     ):
         type_modifier = self._validate_steps_count(start, end, gap)
         start = type_modifier(start)
@@ -1986,7 +1979,7 @@ class ESCollection(Generic[ModelType]):
         if filters:
             args.append(("filters", filters))
 
-        result = self._search(args, use_archive=use_archive)
+        result = self._search(args)
 
         # Convert the histogram into a dictionary
         return {
@@ -2006,7 +1999,6 @@ class ESCollection(Generic[ModelType]):
         mincount=None,
         filters=None,
         access_control=None,
-        use_archive=False,
         field_script=None,
     ):
         if not query:
@@ -2039,7 +2031,7 @@ class ESCollection(Generic[ModelType]):
         if field_script:
             args.append(("field_script", field_script))
 
-        result = self._search(args, use_archive=use_archive)
+        result = self._search(args)
 
         # Convert the histogram into a dictionary
         return {
@@ -2052,7 +2044,6 @@ class ESCollection(Generic[ModelType]):
         query="id:*",
         filters=None,
         access_control=None,
-        use_archive=False,
         field_script=None,
     ):
         if filters is None:
@@ -2076,7 +2067,7 @@ class ESCollection(Generic[ModelType]):
         if field_script:
             args.append(("field_script", field_script))
 
-        result = self._search(args, use_archive=use_archive)
+        result = self._search(args)
         return result["aggregations"][f"{field}_stats"]
 
     def grouped_search(
@@ -2092,7 +2083,6 @@ class ESCollection(Generic[ModelType]):
         filters=None,
         access_control=None,
         as_obj=True,
-        use_archive=False,
         track_total_hits=False,
     ):
         if rows is None:
@@ -2134,7 +2124,7 @@ class ESCollection(Generic[ModelType]):
         if filters:
             args.append(("filters", filters))
 
-        result = self._search(args, use_archive=use_archive, track_total_hits=track_total_hits)
+        result = self._search(args, track_total_hits=track_total_hits)
 
         return {
             "offset": offset,
@@ -2270,6 +2260,75 @@ class ESCollection(Generic[ModelType]):
         else:
             return True
 
+    def _create_ilm_policy(self, ilm_config):
+        """Create or update the ILM policy for this collection.
+
+        Builds an ILM policy with hot (rollover), optional warm (forcemerge),
+        and optional cold phases. No delete phase — retention is handled by
+        the retention cronjob.
+
+        :param ilm_config: The global ILMConfig with rollover settings.
+        """
+        phases: dict[str, Any] = {
+            "hot": {
+                "min_age": "0ms",
+                "actions": {
+                    "rollover": {
+                        "max_age": ilm_config.rollover_max_age,
+                        "max_primary_shard_size": ilm_config.rollover_max_size,
+                    }
+                },
+            }
+        }
+
+        if self.ilm_config and self.ilm_config.warm:
+            phases["warm"] = {
+                "min_age": self.ilm_config.warm,
+                "actions": {
+                    "forcemerge": {"max_num_segments": 1},
+                },
+            }
+
+        if self.ilm_config and self.ilm_config.cold:
+            phases["cold"] = {
+                "min_age": self.ilm_config.cold,
+                "actions": {},
+            }
+
+        policy = {"phases": phases}
+
+        self.with_retries(
+            self.datastore.client.ilm.put_lifecycle,
+            name=f"{self.name}_policy",
+            policy=policy,
+        )
+        logger.info("ILM policy %s_policy created/updated", self.name)
+
+    def _create_index_template(self, ilm_config):
+        """Create or update a composable index template for ILM-managed rollover.
+
+        The template matches '{name}-*' and includes the full ODM mappings
+        so that rollover indices inherit the correct schema.
+
+        :param ilm_config: The global ILMConfig (unused directly but kept for symmetry).
+        """
+        settings = self._get_index_settings()
+        settings["index"]["lifecycle.name"] = f"{self.name}_policy"
+        settings["index"]["lifecycle.rollover_alias"] = self.name
+
+        mappings = self._get_index_mappings()
+
+        self.with_retries(
+            self.datastore.client.indices.put_index_template,
+            name=f"{self.name}_template",
+            index_patterns=[f"{self.name}-*"],
+            template={
+                "settings": settings,
+                "mappings": mappings,
+            },
+        )
+        logger.info("Index template %s_template created/updated", self.name)
+
     def _get_index_settings(self) -> dict:
         default_stub: dict = deepcopy(default_index)
         settings: dict = default_stub.pop("settings", {})
@@ -2386,8 +2445,15 @@ class ESCollection(Generic[ModelType]):
         """This function should test if the collection that you are trying to access does indeed exist
         and should create it if it does not.
 
+        When ILM is configured for this collection, it sets up the ILM policy,
+        composable index template, and bootstraps a rollover alias instead of
+        using the legacy _hot index naming.
+
         :return:
         """
+        if self.ilm_config:
+            return self._ensure_collection_ilm()
+
         # Create HOT index
         if not self.with_retries(self.datastore.client.indices.exists, index=self.name):
             logger.debug("Index %s does not exist. Creating it now...", self.name.upper())
@@ -2430,6 +2496,116 @@ class ESCollection(Generic[ModelType]):
 
         self._check_fields()
 
+    def _ensure_collection_ilm(self):
+        """Bootstrap an ILM-managed collection with rollover alias.
+
+        1. Create/update the ILM policy and composable index template.
+        2. Bootstrap the initial index if needed:
+           - If ILM indices already exist (pattern {name}-0*), skip.
+           - If a legacy _hot index exists, migrate it to {name}-000001.
+           - Otherwise, create {name}-000001 from scratch.
+        """
+        from howler.odm.models.config import config as _config
+
+        ilm_global = _config.datastore.ilm
+
+        # Idempotent: create/update ILM policy and index template
+        self._create_ilm_policy(ilm_global)
+        self._create_index_template(ilm_global)
+
+        ilm_initial_index = f"{self.name}-000001"
+
+        # Check if any ILM-managed index already exists
+        existing_ilm_indices = list(
+            self.with_retries(
+                self.datastore.client.indices.get, index=f"{self.name}-0*", ignore_unavailable=True
+            ).keys()
+        )
+
+        if existing_ilm_indices:
+            # ILM already bootstrapped — ensure the alias exists
+            if not self.with_retries(self.datastore.client.indices.exists_alias, name=self.name):
+                # Find the latest index to set as write index
+                latest = sorted(existing_ilm_indices)[-1]
+                self.with_retries(
+                    self.datastore.client.indices.put_alias,
+                    index=latest,
+                    name=self.name,
+                    is_write_index=True,
+                )
+            logger.debug("ILM collection %s already bootstrapped", self.name.upper())
+        elif self.with_retries(self.datastore.client.indices.exists, index=self.index_name):
+            # Legacy _hot index exists — migrate to ILM
+            logger.info("Migrating %s from legacy _hot index to ILM-managed rollover", self.name.upper())
+
+            # Block writes on the old index
+            self.with_retries(
+                self.datastore.client.indices.put_settings,
+                index=self.index_name,
+                settings=write_block_settings,
+            )
+
+            # Clone the _hot index to the new ILM initial index
+            self._safe_index_copy(self.datastore.client.indices.clone, self.index_name, ilm_initial_index)
+
+            # Apply ILM settings to the new index
+            self.with_retries(
+                self.datastore.client.indices.put_settings,
+                index=ilm_initial_index,
+                settings={
+                    "index.lifecycle.name": f"{self.name}_policy",
+                    "index.lifecycle.rollover_alias": self.name,
+                    "index.blocks.write": None,
+                },
+            )
+
+            # Swap alias: remove old _hot, add new ILM index as write index
+            actions = [
+                {"add": {"index": ilm_initial_index, "alias": self.name, "is_write_index": True}},
+            ]
+
+            # Remove old alias if it points to _hot
+            if self.with_retries(self.datastore.client.indices.exists_alias, index=self.index_name, name=self.name):
+                actions.append({"remove": {"index": self.index_name, "alias": self.name}})
+
+            self.with_retries(self.datastore.client.indices.update_aliases, actions=actions)
+
+            # Unblock writes on the old index (it stays around until manually removed)
+            self.with_retries(
+                self.datastore.client.indices.put_settings,
+                index=self.index_name,
+                settings=write_unblock_settings,
+            )
+
+            # Update index_name to point to the ILM initial index
+            self.index_name = ilm_initial_index
+
+            logger.info("Migration of %s to ILM complete", self.name.upper())
+        else:
+            # Fresh install — create the initial ILM index with alias
+            logger.debug("Creating ILM-managed index %s...", ilm_initial_index)
+            settings = self._get_index_settings()
+            settings["index"]["lifecycle.name"] = f"{self.name}_policy"
+            settings["index"]["lifecycle.rollover_alias"] = self.name
+
+            try:
+                self.with_retries(
+                    self.datastore.client.indices.create,
+                    index=ilm_initial_index,
+                    mappings=self._get_index_mappings(),
+                    settings=settings,
+                    aliases={self.name: {"is_write_index": True}},
+                )
+            except elasticsearch.exceptions.RequestError as e:
+                if "resource_already_exists_exception" not in str(e):
+                    raise
+                logger.warning("ILM index already exists: %s", ilm_initial_index)
+
+            # Update index_name to point to the ILM initial index
+            self.index_name = ilm_initial_index
+
+        self._check_fields()
+
     def _add_fields(self, missing_fields: Dict):
         no_fix = []
         properties = {}
@@ -2467,6 +2643,13 @@ class ESCollection(Generic[ModelType]):
                 **recursive_update(current_template, {"mappings": {"properties": properties}}),
             )
 
+        # When ILM is enabled, also update the composable index template so
+        # future rollover indices inherit the new field mappings.
+        if self.ilm_config:
+            from howler.odm.models.config import config as _config
+
+            self._create_index_template(_config.datastore.ilm)
+
     def wipe(self):
         """This function should completely delete the collection
 

{howler_api-3.4.0.dev912 → howler_api-3.4.0.dev927}/howler/datastore/howler_store.py

@@ -51,7 +51,8 @@ class HowlerDatastore(object):
             )
 
         for _index, _odm in INDEXES:
-            self.ds.register(_index, _odm)
+            ilm_index_config = config.datastore.ilm.indices.get(_index) if config.datastore.ilm.enabled else None
+            self.ds.register(_index, _odm, ilm_config=ilm_index_config)
 
     def __enter__(self):
         return self

{howler_api-3.4.0.dev912 → howler_api-3.4.0.dev927}/howler/datastore/store.py

@@ -166,10 +166,14 @@ class ESStore(object):
             KeyError: If *name* has not been registered via ``register``.
         """
         if not self.validate:
-            return ESCollection(self, name, model_class=self._models[name], validate=self.validate)
+            ilm_cfg = self.__dict__.get("_ilm_configs", {}).get(name)
+            return ESCollection(self, name, model_class=self._models[name], validate=self.validate, ilm_config=ilm_cfg)
 
         if name not in self._collections:
-            self._collections[name] = ESCollection(self, name, model_class=self._models[name], validate=self.validate)
+            ilm_cfg = self.__dict__.get("_ilm_configs", {}).get(name)
+            self._collections[name] = ESCollection(
+                self, name, model_class=self._models[name], validate=self.validate, ilm_config=ilm_cfg
+            )
 
         return self._collections[name]
 
@@ -307,7 +311,7 @@ class ESStore(object):
         """
         return self.client.ping()
 
-    def register(self, name: str, model_class=None):
+    def register(self, name: str, model_class=None, ilm_config=None):
         """Register a collection (index) name and its optional ODM model class.
 
         Args:
@@ -315,6 +319,7 @@ class ESStore(object):
                 and underscores.
             model_class: ODM model class used for validation and serialisation.
                 ``None`` disables model-level validation for this collection.
+            ilm_config: Optional per-index ILM configuration (ILMIndexConfig).
 
         Raises:
             DataStoreException: If *name* contains invalid characters.
@@ -326,6 +331,10 @@ class ESStore(object):
             )
 
         self._models[name] = model_class
+        if ilm_config is not None:
+            if "_ilm_configs" not in self.__dict__:
+                self._ilm_configs: dict = {}
+            self._ilm_configs[name] = ilm_config
 
     def to_pydatemath(self, value):
         """Convert an internal date-math expression to ES date-math syntax.

{howler_api-3.4.0.dev912 → howler_api-3.4.0.dev927}/howler/odm/models/config.py

@@ -93,6 +93,47 @@ class Host(BaseModel):
         return self.__repr__()
 
 
+class ILMIndexConfig(BaseModel):
+    """Per-index ILM phase configuration.
+
+    Controls when an index transitions to warm and cold phases.
+    Values are Elasticsearch age strings (e.g. "30d", "90d").
+    """
+
+    warm: Optional[str] = Field(
+        default=None,
+        description="Min age before the index enters the warm phase (e.g. '30d'). None to skip.",
+    )
+    cold: Optional[str] = Field(
+        default=None,
+        description="Min age before the index enters the cold phase (e.g. '90d'). None to skip.",
+    )
+
+
+class ILMConfig(BaseModel):
+    """Index Lifecycle Management configuration.
+
+    When enabled, Howler uses Elasticsearch ILM policies and rollover aliases
+    to split large indices into time-based segments. This cooperates with the
+    existing retention cronjob — ILM handles rollover and phase transitions,
+    while the retention job handles document deletion.
+    """
+
+    enabled: bool = Field(default=False, description="Enable ILM-based index rollover")
+    rollover_max_age: str = Field(
+        default="30d",
+        description="Maximum age of the write index before rollover (e.g. '30d')",
+    )
+    rollover_max_size: str = Field(
+        default="50gb",
+        description="Maximum primary shard size before rollover (e.g. '50gb')",
+    )
+    indices: dict[str, ILMIndexConfig] = Field(
+        default={},
+        description="Per-index ILM configuration, keyed by collection name (e.g. 'hit')",
+    )
+
+
 class Datastore(BaseModel):
     """Datastore configuration for Howler.
 
@@ -107,6 +148,10 @@ class Datastore(BaseModel):
     type: Literal["elasticsearch"] = Field(
         default="elasticsearch", description="Type of application used for the datastore"
     )
+    ilm: ILMConfig = Field(
+        default_factory=ILMConfig,
+        description="Index Lifecycle Management configuration",
+    )
 
 
 class Logging(BaseModel):

{howler_api-3.4.0.dev912 → howler_api-3.4.0.dev927}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "3.4.0.dev912"
+version = "3.4.0.dev927"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",