howler-api 4.0.0.dev1281__tar.gz → 4.0.0.dev1291__tar.gz

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1281
+Version: 4.0.0.dev1291
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/howler/api/v2/ingest.py

@@ -11,7 +11,7 @@ from howler.common.exceptions import HowlerException, HowlerValueError
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.common.swagger import generate_swagger_docs
-from howler.config import config
+from howler.config import CORRELATION_QUEUE_NAME, config
 from howler.datastore.collection import ESCollection
 from howler.datastore.exceptions import DataStoreException
 from howler.datastore.howler_store import INDEXES
@@ -46,7 +46,7 @@ def _get_ingestion_queue() -> NamedQueue[str]:
 
     if _ingestion_queue is None:
         _ingestion_queue = NamedQueue(
-            "howler.ingestion_queue",
+            CORRELATION_QUEUE_NAME,
             host=config.core.redis.persistent.host,
             port=config.core.redis.persistent.port,
             private=False,

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/howler/common/loader.py

@@ -14,6 +14,7 @@ if TYPE_CHECKING:
 
 
 APP_NAME = os.environ.get("APP_NAME", "howler")
+DATASTORE_INDEX_PREFIX = os.environ.get("HWL_DATASTORE_INDEX_PREFIX", APP_NAME)
 APP_PREFIX = os.environ.get("APP_PREFIX", "hwl")
 USER_TYPES = {"admin", "user", "automation_basic", "automation_advanced", "actionrunner_basic", "actionrunner_advanced"}
 

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/howler/config.py

@@ -24,6 +24,7 @@ HWL_USE_WEBSOCKET_API = os.environ.get("HWL_USE_WEBSOCKET_API", "false").lower()
 HWL_USE_JOB_SYSTEM = os.environ.get("HWL_USE_JOB_SYSTEM", "false").lower() == "true"
 HWL_ENABLE_COVERAGE = os.environ.get("HWL_ENABLE_COVERAGE", "false").lower() == "true"
 HWL_PLUGIN_DIRECTORY = os.environ.get("HWL_PLUGIN_DIRECTORY", "/etc/howler/plugins")
+CORRELATION_QUEUE_NAME = os.environ.get("HWL_CORRELATION_QUEUE_NAME", "howler.ingestion_queue")
 
 
 def get_version() -> str:

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/howler/datastore/collection.py

@@ -20,7 +20,7 @@ from opentelemetry import trace
 
 from howler import odm
 from howler.common.exceptions import HowlerRuntimeError, HowlerValueError, NonRecoverableError
-from howler.common.loader import APP_NAME
+from howler.common.loader import DATASTORE_INDEX_PREFIX
 from howler.common.logging.format import HWL_DATE_FORMAT, HWL_LOG_FORMAT
 from howler.datastore.constants import BACK_MAPPING, TYPE_MAPPING
 from howler.datastore.exceptions import (
@@ -229,7 +229,7 @@ class ESCollection(Generic[ModelType]):
         self._index_list: list[str] = []
 
         self.datastore = datastore
-        self.name = f"{APP_NAME}-{name}"
+        self.name = f"{DATASTORE_INDEX_PREFIX}-{name}"
         self.ilm_config = ilm_config
         self.index_name = f"{self.name}_hot"
         self.model_class = model_class

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/howler/services/correlation_service.py

@@ -16,6 +16,7 @@ from opentelemetry import trace
 from howler.common.exceptions import InvalidDataException, NotFoundException
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
+from howler.config import CORRELATION_QUEUE_NAME
 from howler.odm.models.case import CaseRule, RuleIndexTypes
 from howler.odm.models.config import config
 from howler.remote.datatypes.queues.named import NamedQueue
@@ -181,7 +182,7 @@ def process_batch(record_ids: list[str]) -> int:  # noqa: C901
 def _build_queue() -> NamedQueue[str]:
     """Create the NamedQueue backed by persistent Redis."""
     return NamedQueue(
-        "howler.ingestion_queue",
+        CORRELATION_QUEUE_NAME,
         host=config.core.redis.persistent.host,
         port=config.core.redis.persistent.port,
         private=False,
@@ -210,7 +211,7 @@ def run_worker() -> None:  # pragma: no cover – long-running loop, tested via
                 logger.debug("Processing correlation batch of %d hit(s)", len(batch))
                 try:
                     added = process_batch(batch)
-                    logger.info("Correlation batch complete: %d/%d hit(s) added", added, len(batch))
+                    logger.info("Correlation batch complete: %d case item(s) added for %d record(s)", added, len(batch))
                 except Exception:
                     logger.exception("Error processing correlation batch")
                 finally:

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/howler/services/fuzzy_service.py

@@ -8,7 +8,7 @@ import elasticsearch
 from elasticsearch import Elasticsearch
 
 from howler import odm
-from howler.common.loader import APP_NAME, datastore
+from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.datastore.exceptions import SearchException, SearchRetryException
 from howler.datastore.types import SearchResult
@@ -25,7 +25,7 @@ from howler.odm.base import (
     Text,
     _Field,
 )
-from howler.services.search_service import _normalize_indexes
+from howler.utils.indexes import get_logical_index_name, normalize_indexes
 from howler.utils.str_utils import sanitize_lucene_query
 
 DEFAULT_OFFSET = 0
@@ -369,7 +369,7 @@ def fuzzy_search(
             raise SearchException(f"Invalid index for fuzzy search: {idx}")
 
     client: Elasticsearch = datastore().ds.client
-    parsed_indexes = _normalize_indexes(indexes)
+    parsed_indexes = normalize_indexes(indexes)
 
     query_body = build_fuzzy_query(query, indexes, filters, access_control)
 
@@ -420,7 +420,7 @@ def _format_items_with_score(hits: list[dict[str, Any]]) -> list[dict[str, Any]]
         if source:
             raw_index = hit.get("_index", None)
             if raw_index:
-                source["__index"] = raw_index.replace(f"{APP_NAME}-", "").replace("_hot", "")
+                source["__index"] = get_logical_index_name(raw_index)
             source["_score"] = hit.get("_score", 0)
             items.append(source)
     return items

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/howler/services/search_service.py

@@ -5,7 +5,7 @@ from typing import Any
 import elasticsearch
 from elasticsearch import Elasticsearch
 
-from howler.common.loader import APP_NAME, datastore
+from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.datastore.collection import parse_sort
 from howler.datastore.exceptions import SearchException, SearchRetryException
@@ -13,6 +13,7 @@ from howler.datastore.types import SearchResult
 from howler.helper.search import get_collection, has_access_control
 from howler.odm.models.user import User
 from howler.services import case_service
+from howler.utils.indexes import get_logical_index_name, normalize_indexes
 
 DEFAULT_OFFSET = 0
 DEFAULT_ROW_SIZE = 25
@@ -23,50 +24,6 @@ SCROLL_TIMEOUT = "5m"
 logger = get_logger(__file__)
 
 
-def _normalize_indexes(indexes: str | list[str]) -> str:
-    """Normalizes Elasticsearch index names into a comma-separated string.
-
-    Parses the input indexes and applies naming conventions. Special patterns like
-    wildcards, exclusions, and explicitly formatted indexes are preserved as-is.
-    Regular indexes are formatted with the APP_NAME prefix and '_hot' suffix.
-
-    Args:
-        indexes: A comma-separated string or list of index names to normalize.
-
-    Returns:
-        A comma-separated string of normalized index names ready for Elasticsearch queries.
-
-    Raises:
-        SearchException: If no valid indexes are provided after parsing and stripping whitespace.
-
-    Examples:
-        >>> _normalize_indexes("logs,metrics")
-        "howler-logs_hot,howler-metrics_hot"
-
-        >>> _normalize_indexes(["*", "custom-index"])
-        "*,custom-index"
-
-        >>> _normalize_indexes("alerts, events")
-        "howler-alerts_hot,howler-events_hot"
-    """
-    if isinstance(indexes, str):
-        parsed_indexes = [item.strip() for item in indexes.split(",") if item.strip()]
-    else:
-        parsed_indexes = [item.strip() for item in indexes if item.strip()]
-
-    if not parsed_indexes:
-        raise SearchException("No indexes were provided.")
-
-    normalized_indexes: list[str] = []
-    for index in parsed_indexes:
-        if index in {"*", "_all"} or "-" in index or "*" in index:
-            normalized_indexes.append(index)
-        else:
-            normalized_indexes.append(f"{APP_NAME}-{index}")
-
-    return ",".join(normalized_indexes)
-
-
 def _format_items(hits: list[dict[str, Any]], user_classification: str | None) -> list[dict[str, Any]]:
     """Formats Elasticsearch search hits into a standardized item format.
 
@@ -86,7 +43,7 @@ def _format_items(hits: list[dict[str, Any]], user_classification: str | None) -
             raw_index = hit.get("_index", None)
 
             if raw_index:
-                source["__index"] = raw_index.replace(f"{APP_NAME}-", "").replace("_hot", "")
+                source["__index"] = get_logical_index_name(raw_index)
 
             if source.get("__index") == "case" and user_classification:
                 case_service.filter_case_items_by_classification(source, user_classification)
@@ -137,7 +94,7 @@ def search(  # noqa: C901
     del metadata
 
     client: Elasticsearch = datastore().ds.client
-    parsed_indexes = _normalize_indexes(indexes)
+    parsed_indexes = normalize_indexes(indexes)
 
     if filters is None:
         parsed_filters: list[str] = []

howler_api-4.0.0.dev1291/howler/utils/indexes.py

@@ -0,0 +1,56 @@
+from howler.common.loader import APP_NAME, DATASTORE_INDEX_PREFIX
+from howler.datastore.exceptions import SearchException
+
+
+def normalize_indexes(indexes: str | list[str]) -> str:
+    """Normalizes Elasticsearch index names into a comma-separated string.
+
+    Parses the input indexes and applies naming conventions. Special patterns like
+    wildcards, exclusions, and explicitly formatted indexes are preserved as-is.
+    Regular indexes are formatted with the datastore index prefix and '_hot' suffix.
+
+    Args:
+        indexes: A comma-separated string or list of index names to normalize.
+
+    Returns:
+        A comma-separated string of normalized index names ready for Elasticsearch queries.
+
+    Raises:
+        SearchException: If no valid indexes are provided after parsing and stripping whitespace.
+
+    Examples:
+        >>> normalize_indexes("logs,metrics")
+        "howler-logs_hot,howler-metrics_hot"
+
+        >>> normalize_indexes(["*", "custom-index"])
+        "*,custom-index"
+
+        >>> normalize_indexes("alerts, events")
+        "howler-alerts_hot,howler-events_hot"
+    """
+    if isinstance(indexes, str):
+        parsed_indexes = [item.strip() for item in indexes.split(",") if item.strip()]
+    else:
+        parsed_indexes = [item.strip() for item in indexes if item.strip()]
+
+    if not parsed_indexes:
+        raise SearchException("No indexes were provided.")
+
+    normalized_indexes: list[str] = []
+    for index in parsed_indexes:
+        if index in {"*", "_all"} or "-" in index or "*" in index:
+            normalized_indexes.append(index)
+        else:
+            normalized_indexes.append(f"{DATASTORE_INDEX_PREFIX}-{index}")
+
+    return ",".join(normalized_indexes)
+
+
+def get_logical_index_name(raw_index: str) -> str:
+    """Strip the physical datastore prefix and hot-index suffix from an index name."""
+    for index_prefix in sorted({DATASTORE_INDEX_PREFIX, APP_NAME}, key=len, reverse=True):
+        if raw_index.startswith(f"{index_prefix}-"):
+            raw_index = raw_index.removeprefix(f"{index_prefix}-")
+            break
+
+    return raw_index.removesuffix("_hot")

{howler_api-4.0.0.dev1281 → howler_api-4.0.0.dev1291}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev1281"
+version = "4.0.0.dev1291"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",