howler-api 4.0.0.dev669__tar.gz → 4.0.0.dev679__tar.gz

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev669
+Version: 4.0.0.dev679
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/api/v1/view.py

@@ -261,7 +261,7 @@ def remove_as_favourite(view_id: str, **kwargs):
     """Remove a view from a list of the user's favourites
 
     Variables:
-    id => The id of the view to remove as a favourite
+    view_id => The id of the view to remove as a favourite
 
     Optional Arguments:
     None

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/datastore/collection.py

@@ -270,14 +270,17 @@ class ESCollection(Generic[ModelType]):
         if index is None:
             index = self.index_name
 
+        client = self.datastore.client
+        if request_timeout is not None:
+            client = client.options(request_timeout=request_timeout)
+
         # initial search
         resp = self.with_retries(
-            self.datastore.client.search,
+            client.search,
             index=index,
             query=query,
             scroll=scroll,
             size=size,
-            request_timeout=request_timeout,
             sort=sort,
             _source=source,
         )
@@ -305,16 +308,18 @@ class ESCollection(Generic[ModelType]):
 
         finally:
             if scroll_id:
-                resp = self.with_retries(
-                    self.datastore.client.clear_scroll,
-                    scroll_id=[scroll_id],
-                    ignore=(404,),
-                )
-                if not resp.get("succeeded", False):
-                    logger.warning(
-                        f"Could not clear scroll ID {scroll_id}, there is potential "
-                        "memory leak in you Elastic cluster..."
+                try:
+                    resp = self.with_retries(
+                        self.datastore.client.clear_scroll,
+                        scroll_id=[scroll_id],
                     )
+                    if not resp.get("succeeded", False):
+                        logger.warning(
+                            f"Could not clear scroll ID {scroll_id}, there is potential "
+                            "memory leak in your Elastic cluster..."
+                        )
+                except elasticsearch.exceptions.NotFoundError:
+                    pass
 
     def with_retries(self, func, *args, raise_conflicts=False, **kwargs):
         """This function performs the passed function with the given args and kwargs and reconnect if it fails
@@ -470,7 +475,9 @@ class ESCollection(Generic[ModelType]):
                     raise
 
     def _safe_index_copy(self, copy_function, src, target, settings=None, min_status="yellow"):
-        ret = copy_function(index=src, target=target, settings=settings, request_timeout=60)
+        options_client = self.datastore.client.options(request_timeout=60)
+        timed_function = getattr(options_client.indices, copy_function.__name__)
+        ret = timed_function(index=src, target=target, settings=settings)
         if not ret["acknowledged"]:
             raise DataStoreException(f"Failed to create index {target} from {src}.")
 
@@ -1657,19 +1664,23 @@ class ESCollection(Generic[ModelType]):
 
         # Check if the scroll is finished and close it
         if deep_paging_id is not None and new_deep_paging_id is None:
-            self.with_retries(
-                self.datastore.client.clear_scroll,
-                scroll_id=[deep_paging_id],
-                ignore=(404,),
-            )
+            try:
+                self.with_retries(
+                    self.datastore.client.clear_scroll,
+                    scroll_id=[deep_paging_id],
+                )
+            except elasticsearch.exceptions.NotFoundError:
+                pass
 
         # Check if we can tell from inspection that we have finished the scroll
         if new_deep_paging_id is not None and len(ret_data["items"]) < ret_data["rows"]:
-            self.with_retries(
-                self.datastore.client.clear_scroll,
-                scroll_id=[new_deep_paging_id],
-                ignore=(404,),
-            )
+            try:
+                self.with_retries(
+                    self.datastore.client.clear_scroll,
+                    scroll_id=[new_deep_paging_id],
+                )
+            except elasticsearch.exceptions.NotFoundError:
+                pass
             new_deep_paging_id = None
 
         if new_deep_paging_id is not None:

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/datastore/store.py

@@ -9,7 +9,6 @@ from typing import Any, Optional, cast
 from urllib.parse import urlparse
 
 import elasticsearch
-import elasticsearch.client
 
 from howler.common.logging.format import HWL_DATE_FORMAT, HWL_LOG_FORMAT
 from howler.datastore.collection import ESCollection
@@ -138,7 +137,6 @@ class ESStore(object):
                 max_retries=0,
                 request_timeout=TRANSPORT_TIMEOUT,
             )
-        self.eql = elasticsearch.client.EqlClient(self.client)
         self.archive_access = archive_access
         self.url_path = "elastic"
 
@@ -218,14 +216,12 @@ class ESStore(object):
             max_retries=0,
             request_timeout=TRANSPORT_TIMEOUT,
         )
-        self.eql = elasticsearch.client.EqlClient(self.client)
 
     def close(self):
         self._closed = True
         # Flatten the client object so that attempts to access without reconnecting errors hard
         # But 'cast' it so that mypy and other linters don't think that its normal for client to be None
         self.client = cast(elasticsearch.Elasticsearch, None)
-        self.eql = cast(elasticsearch.client.EqlClient, None)
 
     def get_hosts(self, safe=False):
         if not safe:

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/odm/base.py

@@ -1503,11 +1503,31 @@ def recursive_set_name(field, name, to_parent=False):
         recursive_set_name(field.child_type, name, to_parent=True)
 
 
-def model(index=None, store=None, description=None):
-    """Decorator to create model objects."""
+def model(index=None, store=None, description=None, id_field=None):
+    """Decorator that finalizes a Model subclass for use with the datastore.
+    Assigns metadata to the class (description, id field), validates that all
+    declared field names are legal, recursively sets each field's name, and
+    applies default index/store settings to every field.
+    If ``id_field`` is not provided, it defaults to ``<classname_lower>_id``.
+    Args:
+        index: Default index setting applied to all fields on the model.
+        store: Default store setting applied to all fields on the model.
+        description: Human-readable description of the model.
+        id_field: Name of the field used as the primary key. Defaults to
+            ``<classname_lower>_id`` when not specified.
+    Returns:
+        A class decorator that configures and returns the decorated Model subclass.
+    Raises:
+        HowlerValueError: If any field name fails the ``FIELD_SANITIZER`` regex
+            or appears in ``BANNED_FIELDS``.
+    """
 
     def _finish_model(cls):
         cls._Model__description = description
+        cls._Model__id_field = id_field
+
+        if cls._Model__id_field is None:
+            cls._Model__id_field = f"{cls.__name__.lower()}_id"
 
         for name, field_data in cls.fields().items():
             if not FIELD_SANITIZER.match(name) or name in BANNED_FIELDS:

howler_api-4.0.0.dev679/howler/odm/mixins.py

@@ -0,0 +1,97 @@
+"""Datastore convenience mixins for Howler ODM Model classes.
+
+Provides :class:`DatastoreMixin`, a generic mixin that adds a class-level
+``store`` property (returning a typed :class:`ESCollection`) and instance-level
+``ds`` / ``save`` helpers so that Model subclasses can interact with the
+Elasticsearch datastore without boilerplate.
+"""
+
+from __future__ import annotations
+
+from operator import attrgetter
+from typing import Generic, TypeVar, overload
+
+from howler.common.exceptions import HowlerRuntimeError
+from howler.common.loader import datastore
+from howler.datastore.collection import ESCollection
+from howler.odm.base import Model
+
+ModelType = TypeVar("ModelType", bound=Model)
+
+
+class _ObjectsDescriptor(Generic[ModelType]):
+    """Descriptor that provides class-level-only access to the model's ESCollection.
+
+    Intended to be accessed exclusively via the class (e.g. ``Case.store``).
+    Raises ``AttributeError`` if accessed from an instance to enforce
+    cla  # noqa: D205
+    """
+
+    @overload
+    def __get__(self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
+
+    @overload
+    def __get__(self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
+
+    def __get__(self, obj: ModelType | None, objtype: type[ModelType] | None = None) -> ESCollection[ModelType]:
+        """Return the ESCollection for the owner class.
+
+        Args:
+            obj: The instance the descriptor was accessed from, or ``None``
+                when accessed via the class.
+            objtype: The owner class (e.g. ``Case``, ``Hit``).
+        Returns:
+            ESCollection[ModelType]: The datastore collection for *objtype*.
+        Raises:
+            AttributeError: If accessed from an instance (*obj* is not ``None``)
+                or if *objtype* cannot be determined.
+        """
+        if obj is not None:
+            raise HowlerRuntimeError(
+                f"'{type(obj).__name__}.store' is a class-level property and cannot be accessed from an instance. "
+                f"Use '{type(obj).__name__}.store' instead."
+            )
+
+        if objtype is None:
+            raise HowlerRuntimeError("Cannot resolve owner class for 'store' descriptor.")
+
+        index_name = objtype.__name__.lower()
+        return datastore()[index_name]
+
+
+class DatastoreMixin(Generic[ModelType]):
+    """Mixin that provides convenience datastore access to Model instances.
+
+    Generic over ``ModelType`` so that the ``store`` class property returns a
+    correctly-typed ``ESCollection[ModelType]``.  Adds a ``ds`` property for
+    retrieving the shared datastore connection, a ``store`` class-only property
+    for retrieving the model's ESCollection (raises ``AttributeError`` if accessed
+    from an instance), and a ``save`` method that persists the current model
+    instance using its class name as the index and its configured ID field as the
+    document key.
+    """
+
+    store: _ObjectsDescriptor = _ObjectsDescriptor()
+
+    @property
+    def ds(self):
+        """Return the shared datastore instance.
+
+        Returns:
+            The singleton datastore connection used for all persistence operations.
+        """
+        return datastore()
+
+    def save(self) -> bool:
+        """Persist the current model instance to the datastore.
+
+        Determines the target index from the lowercase class name, extracts the
+        model's ID from the configured ID field, and saves the instance.
+        Returns:
+            bool: True if the save operation succeeded, False otherwise.
+        """
+        index_name = self.__class__.__name__.lower()
+        id_field = self.__class__._Model__id_field  # type: ignore[attr-defined]
+        current_id = attrgetter(id_field)(self)
+
+        return self.ds[index_name].save(current_id, self)

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/odm/models/case.py

@@ -3,6 +3,7 @@ from typing import Any, Optional
 from howler import odm
 from howler.common.exceptions import HowlerValueError
 from howler.odm.constants import Status
+from howler.odm.mixins import DatastoreMixin
 from howler.utils.compat import StrEnum
 
 CASE_ITEM_TYPES = {"observable", "hit", "case", "lead", "reference"}
@@ -92,7 +93,7 @@ class CaseEnrichment(odm.Model):
 
 
 @odm.model(index=True, store=True, description="Case model with path-based items, enrichments, rules, and tasks.")
-class Case(odm.Model):
+class Case(DatastoreMixin["Case"], odm.Model):
     case_id: str = odm.UUID(description="A unique identifier for this case.")
     title: str = odm.Keyword(description="Case title.")
     summary: str = odm.Text(description="Short case summary.")

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/odm/models/hit.py

@@ -2,6 +2,7 @@
 
 from howler import odm
 from howler.common.logging import get_logger
+from howler.odm.mixins import DatastoreMixin
 from howler.odm.models.howler_data import HowlerData
 from howler.odm.models.record import Record
 
@@ -12,8 +13,9 @@ logger = get_logger(__file__)
     index=True,
     store=True,
     description="Howler Outline schema which is an extended version of Elastic Common Schema (ECS)",
+    id_field="howler.id",
 )
-class Hit(Record):
+class Hit(DatastoreMixin["Hit"], Record):
     # Howler extended fields. Deviates from ECS
     howler: HowlerData = odm.Compound(
         HowlerData,

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/odm/models/observable.py

@@ -4,6 +4,7 @@ from howler import odm
 from howler.common.exceptions import HowlerValueError
 from howler.common.logging import get_logger
 from howler.odm.howler_enum import HowlerEnum
+from howler.odm.mixins import DatastoreMixin
 from howler.odm.models.record import Record
 
 logger = get_logger(__file__)
@@ -110,8 +111,9 @@ class ObservableData(odm.Model):
     index=True,
     store=True,
     description="Observable schema which is an extended version of Elastic Common Schema (ECS)",
+    id_field="howler.id",
 )
-class Observable(Record):
+class Observable(DatastoreMixin["Observable"], Record):
     # Howler extended fields. Deviates from ECS
     howler: ObservableData = odm.Compound(
         ObservableData,

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/howler/remote/datatypes/__init__.py

@@ -4,7 +4,7 @@ import json
 import logging
 import os
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
 import redis
@@ -30,7 +30,7 @@ pool: dict[tuple[str, str, bool], redis.BlockingConnectionPool] = {}
 
 
 def now_as_iso():
-    s = datetime.utcfromtimestamp(time.time()).isoformat()
+    s = datetime.now(timezone.utc).replace(tzinfo=None).isoformat()
     return f"{s}Z"
 
 

{howler_api-4.0.0.dev669 → howler_api-4.0.0.dev679}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev669"
+version = "4.0.0.dev679"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",