howler-api 4.0.0.dev1283__tar.gz → 4.0.0.dev1297__tar.gz

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1283
+Version: 4.0.0.dev1297
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/howler/common/loader.py

@@ -14,6 +14,7 @@ if TYPE_CHECKING:
 
 
 APP_NAME = os.environ.get("APP_NAME", "howler")
+DATASTORE_INDEX_PREFIX = os.environ.get("HWL_DATASTORE_INDEX_PREFIX", APP_NAME)
 APP_PREFIX = os.environ.get("APP_PREFIX", "hwl")
 USER_TYPES = {"admin", "user", "automation_basic", "automation_advanced", "actionrunner_basic", "actionrunner_advanced"}
 

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/howler/datastore/collection.py

@@ -20,7 +20,7 @@ from opentelemetry import trace
 
 from howler import odm
 from howler.common.exceptions import HowlerRuntimeError, HowlerValueError, NonRecoverableError
-from howler.common.loader import APP_NAME
+from howler.common.loader import DATASTORE_INDEX_PREFIX
 from howler.common.logging.format import HWL_DATE_FORMAT, HWL_LOG_FORMAT
 from howler.datastore.constants import BACK_MAPPING, TYPE_MAPPING
 from howler.datastore.exceptions import (
@@ -229,7 +229,7 @@ class ESCollection(Generic[ModelType]):
         self._index_list: list[str] = []
 
         self.datastore = datastore
-        self.name = f"{APP_NAME}-{name}"
+        self.name = f"{DATASTORE_INDEX_PREFIX}-{name}"
         self.ilm_config = ilm_config
         self.index_name = f"{self.name}_hot"
         self.model_class = model_class

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/howler/odm/models/howler_data.py

@@ -131,17 +131,17 @@ class Log(odm.Model):
 
 @odm.model(index=True, store=True, description="Hit outline header.")
 class Header(odm.Model):
-    threat: Optional[str] = odm.Optional(odm.Keyword(description="The IP of the threat."))
-    target: Optional[str] = odm.Optional(odm.Keyword(description="The target of the hit."))
+    threat: str | None = odm.Optional(odm.Keyword(description="The IP of the threat."))
+    target: str | None = odm.Optional(odm.Keyword(description="The target of the hit."))
     indicators: list[str] = odm.List(odm.Keyword(description="Indicators of the hit."), default=[])
-    summary: Optional[str] = odm.Optional(odm.Keyword(description="Summary of the hit."))
+    summary: str | None = odm.Optional(odm.Keyword(description="Summary of the hit."))
 
 
 @odm.model(index=True, store=True, description="Fields describing the location where this alert has been retained.")
 class Incident(odm.Model):
     platform: str = odm.Keyword(description="The name of the platform for this incident.")
-    incident_id: Optional[str] = odm.Keyword(description="The ID of the incident.", optional=True)
-    url: Optional[str] = odm.Keyword(description="The url where the incident can be found.", optional=True)
+    incident_id: str | None = odm.Keyword(description="The ID of the incident.", optional=True)
+    url: str | None = odm.Keyword(description="The url where the incident can be found.", optional=True)
 
 
 @odm.model(index=True, store=True, description="Labels for the hit")
@@ -177,7 +177,7 @@ DEFAULT_ASSIGNMENT = "unassigned"
 class HowlerData(odm.Model):
     id: str = odm.UUID(description="A UUID for this hit.")
     analytic: str = odm.CaseInsensitiveKeyword(description="Title of the analytic.")
-    assignment: Optional[str] = odm.Keyword(
+    assignment: str = odm.Keyword(
         description="Unique identifier of the assigned user.",
         default=DEFAULT_ASSIGNMENT,
     )
@@ -191,7 +191,7 @@ class HowlerData(odm.Model):
         default=[],
         description="A list of links associated with this hit.",
     )
-    detection: Optional[str] = odm.Optional(
+    detection: str | None = odm.Optional(
         odm.CaseInsensitiveKeyword(description="The detection that produced this hit.")
     )
     hash: str = odm.HowlerHash(
@@ -219,13 +219,13 @@ class HowlerData(odm.Model):
     score: Optional[float] = odm.Optional(
         odm.Float(description="A score assigned by an enrichment to help prioritize triage.", default=0)
     )
-    status = odm.Enum(values=Status, default=Status.OPEN, description="Status of the hit.")
-    scrutiny = odm.Enum(
+    status: str = odm.Enum(values=Status, default=Status.OPEN, description="Status of the hit.")
+    scrutiny: str = odm.Enum(
         values=Scrutiny,
         default=Scrutiny.UNSEEN,
         description="Level of scrutiny done to this hit.",
     )
-    escalation = odm.Enum(
+    escalation: str = odm.Enum(
         values=Escalation,
         default=Escalation.HIT,
         description="Level of escalation of this hit.",
@@ -235,8 +235,8 @@ class HowlerData(odm.Model):
             description="User selected time for hit expiry",
         )
     )
-    assessment: Optional[str] = odm.Optional(odm.Enum(values=Assessment, description="Assessment of the hit."))
-    rationale: Optional[str] = odm.Optional(
+    assessment: str | None = odm.Optional(odm.Enum(values=Assessment, description="Assessment of the hit."))
+    rationale: str | None = odm.Optional(
         odm.Keyword(
             description=(
                 "The rationale behind the hit assessment. Allows it to be understood and verified by other analysts."
@@ -254,9 +254,9 @@ class HowlerData(odm.Model):
         default=[],
         description="A list of changes to the hit with timestamps and attribution.",
     )
-    monitored: Optional[str] = odm.Optional(odm.Keyword(description="Link to the incident monitoring dashboard."))
-    reported: Optional[str] = odm.Optional(odm.Keyword(description="Link to the incident report."))
-    mitigated: Optional[str] = odm.Optional(odm.Keyword(description="Link to the mitigation record (tool dependent)."))
+    monitored: str | None = odm.Optional(odm.Keyword(description="Link to the incident monitoring dashboard."))
+    reported: str | None = odm.Optional(odm.Keyword(description="Link to the incident report."))
+    mitigated: str | None = odm.Optional(odm.Keyword(description="Link to the mitigation record (tool dependent)."))
     outline: Optional[Header] = odm.Optional(odm.Compound(Header), description="The user specified header of the hit")
     incidents: list[Incident] = odm.List(
         odm.Compound(Incident), default=[], description="Fields describing an incident associated with this alert."

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/howler/services/fuzzy_service.py

@@ -8,7 +8,7 @@ import elasticsearch
 from elasticsearch import Elasticsearch
 
 from howler import odm
-from howler.common.loader import APP_NAME, datastore
+from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.datastore.exceptions import SearchException, SearchRetryException
 from howler.datastore.types import SearchResult
@@ -25,7 +25,7 @@ from howler.odm.base import (
     Text,
     _Field,
 )
-from howler.services.search_service import _normalize_indexes
+from howler.utils.indexes import get_logical_index_name, normalize_indexes
 from howler.utils.str_utils import sanitize_lucene_query
 
 DEFAULT_OFFSET = 0
@@ -369,7 +369,7 @@ def fuzzy_search(
             raise SearchException(f"Invalid index for fuzzy search: {idx}")
 
     client: Elasticsearch = datastore().ds.client
-    parsed_indexes = _normalize_indexes(indexes)
+    parsed_indexes = normalize_indexes(indexes)
 
     query_body = build_fuzzy_query(query, indexes, filters, access_control)
 
@@ -420,7 +420,7 @@ def _format_items_with_score(hits: list[dict[str, Any]]) -> list[dict[str, Any]]
         if source:
             raw_index = hit.get("_index", None)
             if raw_index:
-                source["__index"] = raw_index.replace(f"{APP_NAME}-", "").replace("_hot", "")
+                source["__index"] = get_logical_index_name(raw_index)
             source["_score"] = hit.get("_score", 0)
             items.append(source)
     return items

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/howler/services/hit_service.py

@@ -317,6 +317,18 @@ def convert_hit(  # noqa: C901
             "See howler's documentation for more information."
         )
 
+    if odm.howler.assessment:
+        target_escalation = AssessmentEscalationMap[odm.howler.assessment]
+        if odm.howler.escalation != target_escalation:
+            warnings.append(
+                f"Hits with assessment {odm.howler.assessment} must also have escalation set to {target_escalation}."
+            )
+            odm.howler.escalation = str(target_escalation)
+
+    if odm.howler.escalation in [Escalation.MISS, Escalation.EVIDENCE] and odm.howler.status != Status.RESOLVED:
+        warnings.append("Hits with escalation miss or evidence must also have their status set to resolved.")
+        odm.howler.status = Status.RESOLVED.value
+
     if odm.event:
         odm.event.id = odm.howler.id
         if not odm.event.created:

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/howler/services/search_service.py

@@ -5,7 +5,7 @@ from typing import Any
 import elasticsearch
 from elasticsearch import Elasticsearch
 
-from howler.common.loader import APP_NAME, datastore
+from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.datastore.collection import parse_sort
 from howler.datastore.exceptions import SearchException, SearchRetryException
@@ -13,6 +13,7 @@ from howler.datastore.types import SearchResult
 from howler.helper.search import get_collection, has_access_control
 from howler.odm.models.user import User
 from howler.services import case_service
+from howler.utils.indexes import get_logical_index_name, normalize_indexes
 
 DEFAULT_OFFSET = 0
 DEFAULT_ROW_SIZE = 25
@@ -23,50 +24,6 @@ SCROLL_TIMEOUT = "5m"
 logger = get_logger(__file__)
 
 
-def _normalize_indexes(indexes: str | list[str]) -> str:
-    """Normalizes Elasticsearch index names into a comma-separated string.
-
-    Parses the input indexes and applies naming conventions. Special patterns like
-    wildcards, exclusions, and explicitly formatted indexes are preserved as-is.
-    Regular indexes are formatted with the APP_NAME prefix and '_hot' suffix.
-
-    Args:
-        indexes: A comma-separated string or list of index names to normalize.
-
-    Returns:
-        A comma-separated string of normalized index names ready for Elasticsearch queries.
-
-    Raises:
-        SearchException: If no valid indexes are provided after parsing and stripping whitespace.
-
-    Examples:
-        >>> _normalize_indexes("logs,metrics")
-        "howler-logs_hot,howler-metrics_hot"
-
-        >>> _normalize_indexes(["*", "custom-index"])
-        "*,custom-index"
-
-        >>> _normalize_indexes("alerts, events")
-        "howler-alerts_hot,howler-events_hot"
-    """
-    if isinstance(indexes, str):
-        parsed_indexes = [item.strip() for item in indexes.split(",") if item.strip()]
-    else:
-        parsed_indexes = [item.strip() for item in indexes if item.strip()]
-
-    if not parsed_indexes:
-        raise SearchException("No indexes were provided.")
-
-    normalized_indexes: list[str] = []
-    for index in parsed_indexes:
-        if index in {"*", "_all"} or "-" in index or "*" in index:
-            normalized_indexes.append(index)
-        else:
-            normalized_indexes.append(f"{APP_NAME}-{index}")
-
-    return ",".join(normalized_indexes)
-
-
 def _format_items(hits: list[dict[str, Any]], user_classification: str | None) -> list[dict[str, Any]]:
     """Formats Elasticsearch search hits into a standardized item format.
 
@@ -86,7 +43,7 @@ def _format_items(hits: list[dict[str, Any]], user_classification: str | None) -
             raw_index = hit.get("_index", None)
 
             if raw_index:
-                source["__index"] = raw_index.replace(f"{APP_NAME}-", "").replace("_hot", "")
+                source["__index"] = get_logical_index_name(raw_index)
 
             if source.get("__index") == "case" and user_classification:
                 case_service.filter_case_items_by_classification(source, user_classification)
@@ -137,7 +94,7 @@ def search(  # noqa: C901
     del metadata
 
     client: Elasticsearch = datastore().ds.client
-    parsed_indexes = _normalize_indexes(indexes)
+    parsed_indexes = normalize_indexes(indexes)
 
     if filters is None:
         parsed_filters: list[str] = []

howler_api-4.0.0.dev1297/howler/utils/indexes.py

@@ -0,0 +1,56 @@
+from howler.common.loader import APP_NAME, DATASTORE_INDEX_PREFIX
+from howler.datastore.exceptions import SearchException
+
+
+def normalize_indexes(indexes: str | list[str]) -> str:
+    """Normalizes Elasticsearch index names into a comma-separated string.
+
+    Parses the input indexes and applies naming conventions. Special patterns like
+    wildcards, exclusions, and explicitly formatted indexes are preserved as-is.
+    Regular indexes are formatted with the datastore index prefix and '_hot' suffix.
+
+    Args:
+        indexes: A comma-separated string or list of index names to normalize.
+
+    Returns:
+        A comma-separated string of normalized index names ready for Elasticsearch queries.
+
+    Raises:
+        SearchException: If no valid indexes are provided after parsing and stripping whitespace.
+
+    Examples:
+        >>> normalize_indexes("logs,metrics")
+        "howler-logs_hot,howler-metrics_hot"
+
+        >>> normalize_indexes(["*", "custom-index"])
+        "*,custom-index"
+
+        >>> normalize_indexes("alerts, events")
+        "howler-alerts_hot,howler-events_hot"
+    """
+    if isinstance(indexes, str):
+        parsed_indexes = [item.strip() for item in indexes.split(",") if item.strip()]
+    else:
+        parsed_indexes = [item.strip() for item in indexes if item.strip()]
+
+    if not parsed_indexes:
+        raise SearchException("No indexes were provided.")
+
+    normalized_indexes: list[str] = []
+    for index in parsed_indexes:
+        if index in {"*", "_all"} or "-" in index or "*" in index:
+            normalized_indexes.append(index)
+        else:
+            normalized_indexes.append(f"{DATASTORE_INDEX_PREFIX}-{index}")
+
+    return ",".join(normalized_indexes)
+
+
+def get_logical_index_name(raw_index: str) -> str:
+    """Strip the physical datastore prefix and hot-index suffix from an index name."""
+    for index_prefix in sorted({DATASTORE_INDEX_PREFIX, APP_NAME}, key=len, reverse=True):
+        if raw_index.startswith(f"{index_prefix}-"):
+            raw_index = raw_index.removeprefix(f"{index_prefix}-")
+            break
+
+    return raw_index.removesuffix("_hot")

{howler_api-4.0.0.dev1283 → howler_api-4.0.0.dev1297}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev1283"
+version = "4.0.0.dev1297"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",