howler-api 4.1.0.dev1435__tar.gz → 4.1.0.dev1452__tar.gz

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.1.0.dev1435
+Version: 4.1.0.dev1452
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/howler/api/v1/utils/params.py

@@ -1,7 +1,7 @@
 """Parameter parsing and error handling reused across multiple endpoints."""
 
 import functools
-from typing import Callable, Literal
+from typing import Any, Callable, Literal
 
 from flask import request
 
@@ -10,8 +10,10 @@ from howler.common.exceptions import HowlerInvalidParameterException
 
 REFRESH_ARG_OPTIONS = ["true", "false", "wait_for"]
 
+parser_t = Callable[[str | None], Any]
 
-def parse_parameters(**requested_params: Callable[[str | None], str | None] | Literal["required"] | None):
+
+def parse_parameters(**requested_params: parser_t | Literal["required"] | tuple[parser_t, Literal["required"]] | None):
     """A decorator to parse required parameters from the request args.
 
     Usage:
@@ -32,6 +34,9 @@ def parse_parameters(**requested_params: Callable[[str | None], str | None] | Li
     and the raw string value from the request args will be passed to the endpoint function.
     Alternatively, if the parameter is required but does not need parsing,
     pass "required" as the value for the parameter.
+
+    Parser functions which normally accept None can also be marked as required
+    by passing a tuple of (parser_func, "required") as the value for the parameter.
     """
 
     def decorator(func):
@@ -40,17 +45,17 @@ def parse_parameters(**requested_params: Callable[[str | None], str | None] | Li
         def wrapper(*args, **kwargs):
             try:
                 for param_name, parser in requested_params.items():
+                    required_flag = None
+                    if isinstance(parser, tuple):
+                        parser, required_flag = parser
+
                     raw_value = request.args.get(param_name, None)
 
-                    parsed_value: str | None
-                    if parser == "required":
+                    if parser == "required" or required_flag == "required":
                         if raw_value is None:
                             return bad_request(err=f"Missing required parameter: [{param_name}]")
-                        parsed_value = raw_value
-                    else:
-                        parsed_value = parser(raw_value) if parser else raw_value
 
-                    kwargs[param_name] = parsed_value
+                    kwargs[param_name] = parser(raw_value) if callable(parser) else raw_value
 
             except HowlerInvalidParameterException as e:
                 return bad_request(err=e.message)

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/howler/cronjobs/action_queue_worker.py

@@ -13,7 +13,7 @@ from apscheduler.schedulers.base import BaseScheduler
 from howler.common.logging import get_logger
 from howler.config import config
 from howler.odm.models.action import VALID_TRIGGERS
-from howler.services.action_service import _get_action_queue, process_action_batch
+from howler.services.action_service import TriggeredAction, get_action_queue, process_action_batch
 
 logger = get_logger(__file__)
 
@@ -33,7 +33,7 @@ def run_worker(trigger: str) -> None:  # pragma: no cover – long-running loop,
         logger.info("Action queue worker disabled by configuration, not starting for trigger=%s", trigger)
         return
 
-    queue = _get_action_queue(trigger)
+    queue = get_action_queue(trigger)
     logger.info(
         "Action queue worker started for trigger=%s (batch_size=%d, timeout=%ds)",
         trigger,
@@ -41,7 +41,7 @@ def run_worker(trigger: str) -> None:  # pragma: no cover – long-running loop,
         BATCH_TIMEOUT,
     )
 
-    batch: list[dict] = []
+    batch: list[TriggeredAction] = []
 
     while True:
         try:

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/howler/odm/base.py

@@ -11,14 +11,16 @@ independent data models in python. This gives us:
 
 from __future__ import annotations
 
+import base64
 import copy
 import json
 import re
 import typing
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum as PyEnum
 from enum import EnumMeta
-from typing import TYPE_CHECKING, Callable
+from ipaddress import ip_address
+from typing import TYPE_CHECKING, Callable, Literal
 from typing import Any as _Any
 from venv import logger
 
@@ -125,6 +127,7 @@ class _Field:
         reference=None,
         optional=False,
         deprecated=False,
+        sync=True,
     ):
         self.index = index
         self.store = store
@@ -134,6 +137,7 @@ class _Field:
             self.copyto.append(copyto)
         elif copyto:
             self.copyto.extend(copyto)
+        self.sync = sync
 
         self.name = name
         self.parent_name = None
@@ -1427,34 +1431,36 @@ class Model:
         # attribute assignment
         self.__frozen = True
 
-    def as_primitives(self, hidden_fields=False, strip_null=True) -> dict[str, typing.Any]:
+    def as_primitives(
+        self,
+        hidden_fields=False,
+        strip_null=True,
+        ip_format: Literal["str", "int", "encoded_bytes"] | None = None,
+        timestamp_format: Literal["iso", "posix"] | None = None,
+    ) -> dict[str, typing.Any]:
         """Convert the object back into primitives that can be json serialized."""
         out = {}
 
         fields = self.fields()
         for key, value in self._odm_py_obj.items():
             field_type = fields.get(key, Any)
+            if isinstance(field_type, Optional):
+                field_type = field_type.child_type
+
             if value is not None or (value is None and field_type.default_set):
                 if strip_null and value is None:
                     continue
 
-                if isinstance(value, Model):
-                    out[key] = value.as_primitives(strip_null=strip_null)
-                elif isinstance(value, datetime):
-                    out[key] = value.strftime(DATEFORMAT)
-                elif isinstance(value, TypedMapping):
-                    out[key] = {
-                        k: (v.as_primitives(strip_null=strip_null) if isinstance(v, Model) else v)
-                        for k, v in value.items()
-                    }
-                elif isinstance(value, TypedList):
-                    out[key] = [(v.as_primitives(strip_null=strip_null) if isinstance(v, Model) else v) for v in value]
-                elif isinstance(value, ClassificationObject):
-                    out[key] = str(value)
-                    if hidden_fields:
-                        out.update(value.get_access_control_parts())
-                else:
-                    out[key] = value
+                out[key] = self._as_primitive(
+                    value,
+                    field_type,
+                    strip_null=strip_null,
+                    ip_format=ip_format,
+                    timestamp_format=timestamp_format,
+                )
+
+                if hidden_fields and isinstance(value, ClassificationObject):
+                    out.update(value.get_access_control_parts())
         return out
 
     def json(self):
@@ -1520,6 +1526,72 @@ class Model:
     def __contains__(self, name):
         return name.rstrip("_") in self.fields()
 
+    @staticmethod
+    def _as_primitive(value, field_type, strip_null=True, ip_format=None, timestamp_format=None):
+        if isinstance(value, Model):
+            return value.as_primitives(strip_null=strip_null, ip_format=ip_format, timestamp_format=timestamp_format)
+
+        if isinstance(value, datetime):
+            if timestamp_format == "iso":
+                return value.isoformat()
+
+            if timestamp_format == "posix":
+                return int(value.timestamp())
+
+            if value.tzinfo is not None:
+                value = value.astimezone(timezone.utc)
+            return value.strftime(DATEFORMAT)
+
+        if value is not None and isinstance(field_type, IP):
+            if ip_format == "encoded_bytes":
+                ip_bytes = ip_address(value).packed
+                return base64.b64encode(ip_bytes).decode("utf-8")
+
+            if ip_format == "int":
+                return int(ip_address(value))
+
+            if ip_format == "str":
+                return str(ip_address(value))
+
+            return value
+
+        if isinstance(value, TypedMapping):
+            return {
+                k: (
+                    v.as_primitives(strip_null=strip_null, ip_format=ip_format, timestamp_format=timestamp_format)
+                    if isinstance(v, Model)
+                    else Model._as_primitive(
+                        v,
+                        value.type,
+                        strip_null=strip_null,
+                        ip_format=ip_format,
+                        timestamp_format=timestamp_format,
+                    )
+                )
+                for k, v in value.items()
+            }
+
+        if isinstance(value, TypedList):
+            return [
+                (
+                    v.as_primitives(strip_null=strip_null, ip_format=ip_format, timestamp_format=timestamp_format)
+                    if isinstance(v, Model)
+                    else Model._as_primitive(
+                        v,
+                        value.type,
+                        strip_null=strip_null,
+                        ip_format=ip_format,
+                        timestamp_format=timestamp_format,
+                    )
+                )
+                for v in value
+            ]
+
+        if isinstance(value, ClassificationObject):
+            return str(value)
+
+        return value
+
 
 def recursive_set_name(field, name, to_parent=False):
     if not to_parent:

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/howler/odm/models/case.py

@@ -257,8 +257,16 @@ class Case(DatastoreMixin["Case"], odm.Model):
         )
     )
 
-    def as_primitives(self, hidden_fields=False, strip_null=True) -> dict[str, Any]:
-        result = super().as_primitives(hidden_fields, strip_null)
+    def as_primitives(
+        self,
+        hidden_fields=False,
+        strip_null=True,
+        ip_format: Literal["int", "str", "encoded_bytes"] | None = None,
+        timestamp_format: Literal["iso", "posix"] | None = None,
+    ) -> dict[str, Any]:
+        result = super().as_primitives(
+            hidden_fields, strip_null, ip_format=ip_format, timestamp_format=timestamp_format
+        )
 
         result["__index"] = self.__class__.__name__.lower()
 

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/howler/odm/models/howler_data.py

@@ -185,11 +185,13 @@ class HowlerData(odm.Model):
         odm.Keyword(description="Raw telemetry records associated with this hit."),
         default=[],
         store=False,
+        sync=False,
     )
     links: list[Link] = odm.List(
         odm.Compound(Link),
         default=[],
         description="A list of links associated with this hit.",
+        sync=False,
     )
     detection: str | None = odm.Optional(
         odm.CaseInsensitiveKeyword(description="The detection that produced this hit.")
@@ -248,11 +250,13 @@ class HowlerData(odm.Model):
         odm.Compound(Comment),
         default=[],
         description="A list of comments with timestamps and attribution.",
+        sync=False,
     )
     log: list[Log] = odm.List(
         odm.Compound(Log),
         default=[],
         description="A list of changes to the hit with timestamps and attribution.",
+        sync=False,
     )
     monitored: str | None = odm.Optional(odm.Keyword(description="Link to the incident monitoring dashboard."))
     reported: str | None = odm.Optional(odm.Keyword(description="Link to the incident report."))
@@ -272,5 +276,8 @@ class HowlerData(odm.Model):
         description="Votes relating to the hit",
     )
     dossier: list[Lead] = odm.List(
-        odm.Compound(Lead), default=[], description="A list of leads forming the dossier associated with this hit"
+        odm.Compound(Lead),
+        default=[],
+        description="A list of leads forming the dossier associated with this hit",
+        sync=False,
     )

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/howler/odm/models/record.py

@@ -1,5 +1,5 @@
 # mypy: ignore-errors
-from typing import Any, Optional
+from typing import Any, Literal, Optional
 
 from howler import odm
 from howler.common.logging import get_logger
@@ -342,8 +342,16 @@ class Record(odm.Model):
         )
     )
 
-    def as_primitives(self, hidden_fields=False, strip_null=True) -> dict[str, Any]:
-        result = super().as_primitives(hidden_fields, strip_null)
+    def as_primitives(
+        self,
+        hidden_fields=False,
+        strip_null=True,
+        ip_format: Literal["int", "str", "encoded_bytes"] | None = None,
+        timestamp_format: Literal["iso", "posix"] | None = None,
+    ) -> dict[str, Any]:
+        result = super().as_primitives(
+            hidden_fields, strip_null, ip_format=ip_format, timestamp_format=timestamp_format
+        )
 
         result["__index"] = self.__class__.__name__.lower()
 

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/howler/services/action_service.py

@@ -1,5 +1,6 @@
 import json
-from typing import Any, Optional
+from collections import defaultdict
+from typing import Any, Optional, TypedDict
 
 from flask import Response
 
@@ -18,11 +19,19 @@ from howler.utils.str_utils import sanitize_lucene_query
 
 logger = get_logger(__file__)
 
+
+class TriggeredAction(TypedDict):
+    """Type for triggered actions"""
+
+    hit_ids: list[str]
+    uname: str | None
+
+
 # Per-trigger persistent queues for buffering action execution requests.
-_action_queues: dict[str, NamedQueue[dict]] = {}
+_action_queues: dict[str, NamedQueue[TriggeredAction]] = {}
 
 
-def _get_action_queue(trigger: str) -> NamedQueue[dict]:
+def get_action_queue(trigger: str) -> NamedQueue[TriggeredAction]:
     """Return the action queue for *trigger*, creating it on first use.
 
     Raises:
@@ -65,10 +74,10 @@ def enqueue_action_execution(hit_ids: list[str], trigger: str = "create", user:
         return
 
     try:
-        _get_action_queue(trigger).push(
+        get_action_queue(trigger).push(
             {
                 "hit_ids": hit_ids,
-                "user": user.as_primitives() if user is not None and hasattr(user, "as_primitives") else user,
+                "uname": user.uname if user else None,
             }
         )
     except Exception:
@@ -77,7 +86,7 @@ def enqueue_action_execution(hit_ids: list[str], trigger: str = "create", user:
         bulk_execute_on_query(query, trigger=trigger, user=user)
 
 
-def process_action_batch(trigger: str, items: list[dict]) -> None:
+def process_action_batch(trigger: str, items: list[TriggeredAction]) -> None:
     """Process a batch of queued action execution requests for a single trigger.
 
     Groups items by user and issues a single coalesced
@@ -90,26 +99,23 @@ def process_action_batch(trigger: str, items: list[dict]) -> None:
     if not items:
         return
 
-    groups: dict[str | None, tuple[list[str], Any]] = {}
+    groups: dict[str | None, list[str]] = defaultdict(list)
 
     for item in items:
-        user_data = item.get("user")
-        user_key = user_data["uname"] if isinstance(user_data, dict) and "uname" in user_data else None
-
-        if user_key not in groups:
-            groups[user_key] = ([], user_data)
+        uname = item.get("uname")
 
-        groups[user_key][0].extend(item["hit_ids"])
+        groups[uname].extend(item["hit_ids"])
 
-    for user_key, (hit_ids, user_data) in groups.items():
+    for uname, hit_ids in groups.items():
         # Deduplicate IDs within a batch group
         unique_ids = list(dict.fromkeys(hit_ids))
         query = f"howler.id:({' OR '.join(sanitize_lucene_query(h) for h in unique_ids)})"
 
         try:
-            bulk_execute_on_query(query, trigger=trigger, user=user_data)
+            user = datastore().user.get(uname)
+            bulk_execute_on_query(query, trigger=trigger, user=user)
         except Exception:
-            logger.exception("Error processing action batch for trigger=%s user=%s", trigger, user_key)
+            logger.exception("Error processing action batch for trigger=%s user=%s", trigger, uname)
 
 
 def validate_action(new_action: Any) -> Optional[Response]:  # noqa: C901

{howler_api-4.1.0.dev1435 → howler_api-4.1.0.dev1452}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.1.0.dev1435"
+version = "4.1.0.dev1452"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",