howler-api 4.1.0.dev1626__tar.gz → 4.1.0.dev1635__tar.gz

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.1.0.dev1626
+Version: 4.1.0.dev1635
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Topic :: Software Development :: Libraries
 Requires-Dist: apscheduler (==3.11.3)
 Requires-Dist: authlib (>=1.6.0,<2.0.0)

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/howler/api/v1/hit.py

@@ -27,7 +27,8 @@ from howler.datastore.exceptions import DataStoreException, VersionConflictExcep
 from howler.datastore.operations import OdmHelper, OdmUpdateOperation
 from howler.helper.workflow import WorkflowException
 from howler.odm.models.hit import Hit
-from howler.odm.models.howler_data import Comment, HitOperationType, HitStatusTransition
+from howler.odm.models.howler_data import Comment, HitStatusTransition
+from howler.odm.models.log import LogOperationType
 from howler.odm.models.user import User
 from howler.security.login import api_login
 from howler.security.utils import is_classification_accessible, validate_bulk_operation_targets
@@ -407,7 +408,7 @@ def update_hit(
                     "explanation": f"Hit updated by {user.uname}\n\n" + "\n".join(explanation),
                     "new_value": "N/A",
                     "previous_value": "None",
-                    "type": HitOperationType.APPENDED,
+                    "type": LogOperationType.APPENDED,
                     "user": user.uname,
                 },
                 silent=True,

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/howler/cronjobs/retention.py

@@ -9,8 +9,13 @@ from pytz import timezone
 from howler.common.logging import get_logger
 from howler.config import DEBUG, config
 from howler.datastore.howler_store import HowlerDatastore
+from howler.datastore.operations import OdmHelper
+from howler.odm.models.case import CaseItemTypes
+from howler.odm.models.hit import Hit
+from howler.services import case_service
 
 logger = get_logger(__file__)
+hit_helper = OdmHelper(Hit)
 
 
 def execute():
@@ -25,9 +30,7 @@ def execute():
 
     ds = datastore()
 
-    ds.hit.delete_by_query(f"event.created:{{* TO {cutoff}}} OR howler.expiry:{{* TO now}}")
-
-    ds.hit.commit()
+    _delete_matching_hits(ds, f"event.created:{{* TO {cutoff}}} OR howler.expiry:{{* TO now}}")
 
     logger.debug("Deletion complete")
 
@@ -82,8 +85,7 @@ def _execute_rules(ds: HowlerDatastore) -> None:
 
         try:
             count = ds.hit.count(combined_query, filters=None).get("count", -1)
-            ds.hit.delete_by_query(combined_query)
-            ds.hit.commit()
+            _delete_matching_hits(ds, combined_query)
             logger.debug("Retention rule '%s' complete", rule.name)
             result = {"rule": rule.name, "status": "ok", "deleted": count, "cutoff": cutoff}
         except Exception:
@@ -98,6 +100,26 @@ def _execute_rules(ds: HowlerDatastore) -> None:
         logger.info("Rule result: '%s'", result)
 
 
+def _delete_matching_hits(ds: HowlerDatastore, query: str) -> None:
+    """Remove matching hits after cleaning their references from cases and hits."""
+    hit_ids = {hit.howler.id for hit in ds.hit.stream_search(query)}
+
+    if hit_ids:
+        id_query = " OR ".join(hit_ids)
+        for case in ds.case.stream_search(f"items.value:({id_query})"):
+            item_ids = [item.id for item in case.items if item.type == CaseItemTypes.HIT and item.value in hit_ids]
+            if item_ids:
+                case_service.remove_case_items(case, item_ids)
+
+        ds.hit.update_by_query(
+            f"howler.related:({id_query})",
+            [hit_helper.list_remove("howler.related", hit_id, silent=True) for hit_id in hit_ids],
+        )
+
+    ds.hit.delete_by_query(query)
+    ds.hit.commit()
+
+
 def _remove_analytics_without_hits(ds: HowlerDatastore):
     matched_analytics = _find_analytics_with_hits(ds)
 

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/howler/odm/models/event.py

@@ -1,10 +1,10 @@
 # mypy: ignore-errors
 
 from howler import odm
-from howler.common.exceptions import HowlerValueError
 from howler.common.logging import get_logger
 from howler.odm.howler_enum import HowlerEnum
 from howler.odm.mixins import DatastoreMixin
+from howler.odm.models.log import Log
 from howler.odm.models.record import Record
 
 logger = get_logger(__file__)
@@ -30,26 +30,6 @@ class Comment(odm.Model):
     )
 
 
-@odm.model(index=True, store=True, description="Log definition.")
-class Log(odm.Model):
-    timestamp = odm.Date(description="Timestamp at which the Log event took place.")
-    key = odm.Optional(odm.Keyword(description="The key whose value changed."))
-    explanation = odm.Optional(odm.Text(description="A manual description of the changes made."))
-    new_value = odm.Optional(odm.Keyword(description="The value the key is changing to."))
-    previous_value = odm.Optional(odm.Keyword(description="The value the key is changing from."))
-    user = odm.Keyword(description="User ID who created the log event.")
-
-    def __init__(self, data: dict = None, *args, **kwargs):
-        if "explanation" not in data:
-            required_keys = {"key", "new_value", "previous_value"}
-            if required_keys.intersection(set(data.keys())) != required_keys:
-                raise HowlerValueError(
-                    f"If no explanation provided, you must provide the following values: {','.join(required_keys)}"
-                )
-
-        super().__init__(data, *args, **kwargs)
-
-
 DEFAULT_LABELS = {"assignments": [], "generic": []}
 
 

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/howler/odm/models/howler_data.py

@@ -3,10 +3,10 @@ from datetime import datetime
 from typing import Optional
 
 from howler import odm
-from howler.common.exceptions import HowlerValueError
 from howler.odm.constants import Status
 from howler.odm.howler_enum import HowlerEnum
 from howler.odm.models.lead import Lead
+from howler.odm.models.log import Log
 
 
 class Scrutiny(str, HowlerEnum):
@@ -31,12 +31,6 @@ class HitStatusTransition(str, HowlerEnum):
     DEMOTE = "demote"
 
 
-class HitOperationType(str, HowlerEnum):
-    APPENDED = "appended"
-    REMOVED = "removed"
-    SET = "set"
-
-
 class Escalation(str, HowlerEnum):
     MISS = "miss"
     HIT = "hit"
@@ -107,28 +101,6 @@ class Comment(odm.Model):
     )
 
 
-@odm.model(index=True, store=True, description="Log definition.")
-class Log(odm.Model):
-    timestamp = odm.Date(description="Timestamp at which the Log event took place.")
-    key = odm.Optional(odm.Keyword(description="The key whose value changed."))
-    explanation = odm.Optional(odm.Text(description="A manual description of the changes made."))
-    previous_version = odm.Optional(odm.Keyword(description="The version this action was applied to."))
-    new_value = odm.Optional(odm.Keyword(description="The value the key is changing to."))
-    type = odm.Optional(odm.Enum(values=HitOperationType, description="The operation performed on the value."))
-    previous_value = odm.Optional(odm.Keyword(description="The value the key is changing from."))
-    user = odm.Keyword(description="User ID who created the log event.")
-
-    def __init__(self, data: dict = None, *args, **kwargs):
-        if "explanation" not in data:
-            required_keys = {"key", "new_value", "type", "previous_value"}
-            if required_keys.intersection(set(data.keys())) != required_keys:
-                raise HowlerValueError(
-                    f"If no explanation provided, you must provide the following values: {','.join(required_keys)}"
-                )
-
-        super().__init__(data, *args, **kwargs)
-
-
 @odm.model(index=True, store=True, description="Hit outline header.")
 class Header(odm.Model):
     threat: str | None = odm.Optional(odm.Keyword(description="The IP of the threat."))

howler_api-4.1.0.dev1635/howler/odm/models/log.py

@@ -0,0 +1,31 @@
+from howler import odm
+from howler.common.exceptions import HowlerValueError
+from howler.odm.howler_enum import HowlerEnum
+
+
+class LogOperationType(str, HowlerEnum):
+    APPENDED = "appended"
+    REMOVED = "removed"
+    SET = "set"
+
+
+@odm.model(index=True, store=True, description="Log definition.")  # noqa: F821
+class Log(odm.Model):
+    timestamp = odm.Date(description="Timestamp at which the Log event took place.")
+    key = odm.Optional(odm.Keyword(description="The key whose value changed."))
+    explanation = odm.Optional(odm.Text(description="A manual description of the changes made."))
+    previous_version = odm.Optional(odm.Keyword(description="The version this action was applied to."))
+    new_value = odm.Optional(odm.Keyword(description="The value the key is changing to."))
+    type = odm.Optional(odm.Enum(values=LogOperationType, description="The operation performed on the value."))
+    previous_value = odm.Optional(odm.Keyword(description="The value the key is changing from."))
+    user = odm.Keyword(description="User ID who created the log event.")
+
+    def __init__(self, data: dict = None, *args, **kwargs):
+        if "explanation" not in data:
+            required_keys = {"key", "new_value", "previous_value"}
+            if required_keys.intersection(set(data.keys())) != required_keys:
+                raise HowlerValueError(
+                    f"If no explanation provided, you must provide the following values: {','.join(required_keys)}"
+                )
+
+        super().__init__(data, *args, **kwargs)

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/howler/services/case_service.py

@@ -26,6 +26,7 @@ from howler.odm.models.case import Case, CaseItem, CaseItemTypes, CaseLog, CaseR
 from howler.odm.models.ecs.related import Related
 from howler.odm.models.event import Event
 from howler.odm.models.hit import Hit
+from howler.odm.models.log import Log
 from howler.odm.models.user import User
 from howler.security.utils import is_classification_accessible
 from howler.services import comms_service, event_service, hit_service
@@ -788,7 +789,7 @@ def remove_case_items(  # noqa: C901
     case.items = [item for item in case.items if item.id not in ids_to_remove]
 
     for backing_obj, backing_obj_version in backing_objs:
-        remove_backreference(backing_obj, case.case_id)
+        remove_backreference(backing_obj, case.case_id, user=user)
         backing_obj.save(version=backing_obj_version)
 
     recompute_case_metadata(case)
@@ -845,7 +846,7 @@ def append_hit(
 
     case.items.append(item)
 
-    add_backreference(hit, case.case_id)
+    add_backreference(hit, case.case_id, user=user)
     hit.save(version=hit_version)
 
     recompute_case_metadata(case)
@@ -889,7 +890,7 @@ def append_event(
 
     case.items.append(item)
 
-    add_backreference(event, case.case_id)
+    add_backreference(event, case.case_id, user=user)
     event.save(version=event_version)
 
     recompute_case_metadata(case)
@@ -996,7 +997,11 @@ def recompute_case_metadata(case: Case) -> None:  # noqa: C901
     case.indicators = sorted(indicators)
 
 
-def add_backreference(backing_obj: Hit | Event | None, case_id: str) -> bool:
+def add_backreference(
+    backing_obj: Hit | Event | None,
+    case_id: str,
+    user: User | None = None,
+) -> bool:
     """Add a back-reference from a hit or event to a case, in memory only.
 
     Records the case ID in the backing object's ``howler.related`` list so
@@ -1023,12 +1028,23 @@ def add_backreference(backing_obj: Hit | Event | None, case_id: str) -> bool:
         return False
 
     backing_obj.howler.related.append(case_id)
+    backing_obj.howler.log.append(
+        Log(
+            {
+                "timestamp": "NOW",
+                "explanation": f"Added to case {case_id}",
+                "user": user.uname if user else "system",
+            }
+        )
+    )
+
     return True
 
 
 def remove_backreference(
     backing_obj: Hit | Event | None,
     case_id: str,
+    user: User | None = None,
 ):
     """Remove a back-reference from a hit or event to a case, in memory only.
 
@@ -1052,6 +1068,16 @@ def remove_backreference(
     if case_id in backing_obj.howler.related:
         backing_obj.howler.related.remove(case_id)
 
+        backing_obj.howler.log.append(
+            Log(
+                {
+                    "timestamp": "NOW",
+                    "explanation": f"Removed from case {case_id}",
+                    "user": user.uname if user else "system",
+                }
+            )
+        )
+
 
 def rename_case_item(
     case: str | Case | None,

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/howler/services/hit_service.py

@@ -29,7 +29,8 @@ from howler.helper.hit import (
 from howler.helper.workflow import Transition, Workflow
 from howler.odm.models.ecs.event import ECSEvent
 from howler.odm.models.hit import Hit
-from howler.odm.models.howler_data import HitOperationType, HitStatusTransition, Log, Status
+from howler.odm.models.howler_data import HitStatusTransition, Status
+from howler.odm.models.log import Log, LogOperationType
 from howler.odm.models.user import User
 from howler.security.utils import is_classification_accessible
 from howler.services import action_service, analytic_service, dossier_service, overview_service, template_service
@@ -625,16 +626,16 @@ def _update_hit(
 
         if is_list:
             operation_type = (
-                HitOperationType.APPENDED
+                LogOperationType.APPENDED
                 if operation.operation
                 in (
                     ESCollection.UPDATE_APPEND,
                     ESCollection.UPDATE_APPEND_IF_MISSING,
                 )
-                else HitOperationType.REMOVED
+                else LogOperationType.REMOVED
             )
         else:
-            operation_type = HitOperationType.SET
+            operation_type = LogOperationType.SET
 
         logger.debug("%s - %s - %s -> %s", hit_id, operation.key, previous_value, operation.value)
         final_operations.append(operation)

{howler_api-4.1.0.dev1626 → howler_api-4.1.0.dev1635}/pyproject.toml

@@ -244,7 +244,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.1.0.dev1626"
+version = "4.1.0.dev1635"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",