howler-api 3.0.0.dev374__py3-none-any.whl

howler/datastore/constants.py

@@ -0,0 +1,119 @@
+from howler.odm import (
+    IP,
+    MAC,
+    MD5,
+    SHA1,
+    SHA256,
+    URI,
+    UUID,
+    Any,
+    Boolean,
+    CaseInsensitiveKeyword,
+    Classification,
+    ClassificationString,
+    Date,
+    Domain,
+    Email,
+    Enum,
+    FlattenedObject,
+    Float,
+    HowlerHash,
+    Integer,
+    Json,
+    Keyword,
+    Long,
+    LowerKeyword,
+    PhoneNumber,
+    Platform,
+    Processor,
+    SSDeepHash,
+    Text,
+    UpperKeyword,
+    URIPath,
+    ValidatedKeyword,
+)
+
+# Simple types can be resolved by a direct mapping
+BASE_TYPE_MAPPING = {
+    Keyword: "keyword",
+    Boolean: "boolean",
+    Integer: "integer",
+    Long: "long",
+    Float: "float",
+    Date: "date",
+    Text: "text",
+    Classification: "keyword",
+    ClassificationString: "keyword",
+    Enum: "keyword",
+    UUID: "keyword",
+    IP: "ip",
+    Domain: "keyword",
+    Email: "keyword",
+    URI: "keyword",
+    URIPath: "keyword",
+    MAC: "keyword",
+    PhoneNumber: "keyword",
+    SSDeepHash: "text",
+    SHA1: "keyword",
+    SHA256: "keyword",
+    HowlerHash: "keyword",
+    MD5: "keyword",
+    Platform: "keyword",
+    Processor: "keyword",
+    FlattenedObject: "nested",
+    Any: "keyword",
+    UpperKeyword: "keyword",
+    LowerKeyword: "keyword",
+    Json: "keyword",
+    ValidatedKeyword: "keyword",
+    CaseInsensitiveKeyword: "keyword",
+}
+
+TYPE_MAPPING = {_class.__name__: mapping for _class, mapping in BASE_TYPE_MAPPING.items()}
+
+ANALYZER_MAPPING = {
+    SSDeepHash: "text_fuzzy",
+}
+
+NORMALIZER_MAPPING = {
+    SHA1: "lowercase_normalizer",
+    SHA256: "lowercase_normalizer",
+    HowlerHash: "lowercase_normalizer",
+    MD5: "lowercase_normalizer",
+    CaseInsensitiveKeyword: "lowercase_normalizer",
+}
+
+# TODO: We might want to use custom analyzers for Classification and Enum and not create special backmapping cases
+BACK_MAPPING = {
+    v: k
+    for k, v in BASE_TYPE_MAPPING.items()
+    if k
+    not in [
+        Enum,
+        Classification,
+        UUID,
+        IP,
+        Domain,
+        URI,
+        URIPath,
+        MAC,
+        PhoneNumber,
+        SSDeepHash,
+        Email,
+        SHA1,
+        SHA256,
+        HowlerHash,
+        MD5,
+        Platform,
+        Processor,
+        ClassificationString,
+        Any,
+        UpperKeyword,
+        LowerKeyword,
+        CaseInsensitiveKeyword,
+        Json,
+        ValidatedKeyword,
+    ]
+}
+
+BACK_MAPPING.update({x: Keyword for x in set(ANALYZER_MAPPING.values())})

howler/datastore/exceptions.py

@@ -0,0 +1,41 @@
+from typing import Iterable, Optional
+
+from elasticsearch.helpers.errors import ScanError
+
+from howler.common.exceptions import HowlerException, HowlerKeyError
+
+
+class SearchRetryException(HowlerException):
+    pass
+
+
+class DataStoreException(HowlerException):
+    pass
+
+
+class SearchException(HowlerException):
+    pass
+
+
+class SearchDepthException(HowlerException):
+    pass
+
+
+class ILMException(HowlerException):
+    pass
+
+
+class VersionConflictException(HowlerException):
+    pass
+
+
+class MultiKeyError(HowlerKeyError):
+    def __init__(self, keys: Iterable[str], partial_output):
+        super().__init__(str(keys))
+        self.keys = set(keys)
+        self.partial_output = partial_output
+
+
+class HowlerScanError(HowlerException, ScanError):
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else ScanError(message))

howler/datastore/howler_store.py

@@ -0,0 +1,105 @@
+from typing import TYPE_CHECKING
+
+from howler.common.exceptions import HowlerAttributeError
+from howler.datastore.collection import ESCollection, logger
+from howler.odm.models.action import Action
+from howler.odm.models.analytic import Analytic
+from howler.odm.models.dossier import Dossier
+from howler.odm.models.hit import Hit
+from howler.odm.models.overview import Overview
+from howler.odm.models.template import Template
+from howler.odm.models.user import User
+from howler.odm.models.view import View
+from howler.plugins import get_plugins
+
+if TYPE_CHECKING:
+    from howler.datastore.store import ESStore
+
+INDEXES = [
+    ("hit", Hit),
+    ("template", Template),
+    ("overview", Overview),
+    ("analytic", Analytic),
+    ("action", Action),
+    ("user", User),
+    ("view", View),
+    ("dossier", Dossier),
+    ("user_avatar", None),
+]
+
+
+class HowlerDatastore(object):
+    def __init__(self, datastore_object: "ESStore"):
+        self.ds = datastore_object
+
+        for plugin in get_plugins():
+            for _index, _odm in INDEXES:
+                if _odm is None:
+                    continue
+
+                if modify_odm := plugin.modules.odm.modify_odm.get(_index):
+                    logger.info("Modifying %s odm with function from plugin %s", _index, plugin.name)
+                    modify_odm(_odm)
+
+        for _index, _odm in INDEXES:
+            self.ds.register(_index, _odm)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.ds.close()
+
+    def stop_model_validation(self):
+        self.ds.validate = False
+
+    def start_model_validation(self):
+        self.ds.validate = True
+
+    def enable_archive_access(self):
+        self.ds.archive_access = True
+
+    def disable_archive_access(self):
+        self.ds.archive_access = False
+
+    @property
+    def hit(self) -> ESCollection[Hit]:
+        return self.ds.hit
+
+    @property
+    def template(self) -> ESCollection[Template]:
+        return self.ds.template
+
+    @property
+    def overview(self) -> ESCollection[Overview]:
+        return self.ds.overview
+
+    @property
+    def view(self) -> ESCollection[View]:
+        return self.ds.view
+
+    @property
+    def analytic(self) -> ESCollection[Analytic]:
+        return self.ds.analytic
+
+    @property
+    def action(self) -> ESCollection[Action]:
+        return self.ds.action
+
+    @property
+    def user(self) -> ESCollection[User]:
+        return self.ds.user
+
+    @property
+    def dossier(self) -> ESCollection[Dossier]:
+        return self.ds.dossier
+
+    @property
+    def user_avatar(self) -> ESCollection:
+        return self.ds.user_avatar
+
+    def get_collection(self, collection_name: str) -> ESCollection:
+        if collection_name in self.ds.get_models():
+            return getattr(self, collection_name)
+        else:
+            raise HowlerAttributeError(f"Collection {collection_name} does not exist.")

howler/datastore/migrations/fix_process.py

@@ -0,0 +1,41 @@
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+
+logger = get_logger(__file__)
+
+
+def migrate():
+    logger.info("Checking for migration preconditions")
+
+    collection = datastore().hit
+
+    result = collection.search("_exists_:process.parent.start", as_obj=False, track_total_hits=True, rows=0)
+
+    if result["total"] > 0:
+        logger.info("Preconditions met, continuing.")
+
+        db_size = collection.search("howler.id:*", track_total_hits=True, rows=0)["total"]
+        logger.info(f"Database size pre-migration: {db_size}")
+    else:
+        logger.info("Preconditions not met, stopping")
+        return
+
+    logger.info(f"We will delete {result['total']} hits. Continue?")
+    result = input("y/[n]")
+
+    if result.lower() != "y":
+        logger.warning("Did not receive an OK, stopping")
+        return
+
+    logger.info("Deleting...")
+    collection.delete_by_query("_exists_:process.parent.start")
+    collection.commit()
+
+    db_size_after = collection.search("howler.id:*", track_total_hits=True, rows=0)["total"]
+    logger.info(f"Database size post-migration: {db_size_after}")
+
+    logger.info("Migration complete")
+
+
+if __name__ == "__main__":
+    migrate()

howler/datastore/operations.py

@@ -0,0 +1,130 @@
+from typing import Any, Iterator, Optional, Type
+
+from howler.common.exceptions import HowlerTypeError, HowlerValueError
+from howler.datastore.collection import ESCollection
+from howler.odm.base import Model
+
+
+class OdmHelper:
+    def __init__(self, obj: Optional[Type[Model]] = None) -> None:
+        self.model_name = obj.__name__ if obj else None
+        self.valid_fields = list(obj.flat_fields().keys()) if obj else None
+        self.fields = obj.flat_fields() if obj else {}
+
+    def list_add(
+        self,
+        key: str,
+        value: Any,
+        explanation: Optional[str] = None,
+        silent: bool = False,
+        if_missing: bool = False,
+    ):
+        if self.valid_fields and not any(
+            field for field in self.valid_fields if field.startswith(key) and self.fields[field].multivalued
+        ):
+            raise HowlerValueError(f"Key {key} not found in {self.model_name}")
+
+        return OdmUpdateOperation(
+            ESCollection.UPDATE_APPEND_IF_MISSING if if_missing else ESCollection.UPDATE_APPEND,
+            key,
+            value,
+            explanation,
+            silent,
+        )
+
+    def list_remove(
+        self,
+        key: str,
+        value: Any,
+        explanation: Optional[str] = None,
+        silent: bool = False,
+    ):
+        if self.valid_fields and not any(
+            field for field in self.valid_fields if field.startswith(key) and self.fields[field].multivalued
+        ):
+            raise HowlerValueError(f"Key {key} not found in {self.model_name}")
+
+        return OdmUpdateOperation(
+            ESCollection.UPDATE_REMOVE,
+            key,
+            value,
+            explanation,
+            silent,
+        )
+
+    def update(
+        self,
+        key: str,
+        value: Any,
+        explanation: Optional[str] = None,
+        silent: bool = False,
+    ):
+        if self.valid_fields and key not in self.valid_fields:
+            raise HowlerValueError(f"Key {key} not found in {self.model_name}")
+
+        return OdmUpdateOperation(
+            ESCollection.UPDATE_SET,
+            key,
+            value,
+            explanation,
+            silent,
+        )
+
+
+class OdmUpdateOperation:
+    """Provide typing for ODM update operations.
+
+    Attributes:
+        operation: A string defining which operation to perform.
+        key: A string defining which value on the ODM to update.
+        value: The new value to set.
+    """
+
+    __slots__ = ["operation", "key", "value", "explanation", "silent"]
+
+    operation: str
+    key: str
+    value: Any
+    explanation: Optional[str]
+    silent: bool
+
+    def __init__(
+        self,
+        operation: str,
+        key: str,
+        value: Any,
+        explanation: Optional[str] = None,
+        silent: bool = False,
+    ) -> None:
+        if operation not in ESCollection.UPDATE_OPERATIONS:
+            raise HowlerValueError(f"Operation {operation} not found in ESCollection.UPDATE_OPERATIONS")
+
+        self.operation = operation
+
+        if key is not None and not isinstance(key, str):
+            raise HowlerTypeError("Key must be of type str")
+        self.key = key
+        self.value = value
+
+        if explanation is not None and not isinstance(explanation, str):
+            raise HowlerTypeError("Explanation must be of type str")
+        self.explanation = explanation
+        self.silent = silent
+
+    def __iter__(self) -> Iterator[Any]:
+        """Returns generator function for iterating over slots.
+
+        Enables object destructuring:
+        operation, key, value = odmOperation
+        """
+        for slot in self.__slots__:
+            if slot in ["explanation", "silent"]:
+                continue
+
+            yield self.__getattribute__(slot)
+
+    def __repr__(self) -> str:
+        return (
+            f"OdmUpdateOperation: operation={self.operation}, key={self.key}, "
+            + f"value={self.value}, explanation={self.explanation}"
+        )

howler/datastore/schemas.py

@@ -0,0 +1,90 @@
+default_index = {
+    "settings": {
+        "analysis": {
+            "filter": {
+                "text_ws_dsplit": {
+                    "type": "pattern_replace",
+                    "pattern": r"(\.)",
+                    "replacement": " ",
+                }
+            },
+            "analyzer": {
+                "string_ci": {
+                    "type": "custom",
+                    "tokenizer": "keyword",
+                    "filter": ["lowercase"],
+                },
+                "text_fuzzy": {
+                    "type": "pattern",
+                    "pattern": r"\s*:\s*",
+                    "lowercase": False,
+                },
+                "text_whitespace": {"type": "whitespace"},
+                "text_ws_dsplit": {
+                    "type": "custom",
+                    "tokenizer": "whitespace",
+                    "filters": ["text_ws_dsplit"],
+                },
+            },
+            "normalizer": {
+                "lowercase_normalizer": {
+                    "type": "custom",
+                    "char_filter": [],
+                    "filter": ["lowercase"],
+                }
+            },
+        }
+    },
+    "mappings": {},
+}
+
+default_mapping = {
+    "dynamic": True,
+    "properties": {
+        "__text__": {"type": "text"},
+    },
+}
+
+default_dynamic_strings = {
+    "strings_as_keywords": {
+        "match_mapping_type": "string",
+        "mapping": {"type": "keyword", "ignore_above": 8191},
+    }
+}
+
+default_dynamic_templates = [
+    {"int": {"path_match": "*_i", "mapping": {"type": "integer", "store": True}}},
+    {"ints": {"path_match": "*_is", "mapping": {"type": "integer", "store": True}}},
+    {"long": {"path_match": "*_l", "mapping": {"type": "long", "store": True}}},
+    {"longs": {"path_match": "*_ls", "mapping": {"type": "long", "store": True}}},
+    {"double": {"path_match": "*_d", "mapping": {"type": "float", "store": True}}},
+    {"doubles": {"path_match": "*_ds", "mapping": {"type": "float", "store": True}}},
+    {"float": {"path_match": "*_f", "mapping": {"type": "float", "store": True}}},
+    {"floats": {"path_match": "*_fs", "mapping": {"type": "float", "store": True}}},
+    {"string": {"path_match": "*_s", "mapping": {"type": "keyword", "store": True}}},
+    {"strings": {"path_match": "*_ss", "mapping": {"type": "keyword", "store": True}}},
+    {"text": {"path_match": "*_t", "mapping": {"type": "text", "store": True}}},
+    {"texts": {"path_match": "*_ts", "mapping": {"type": "text", "store": True}}},
+    {"boolean": {"path_match": "*_b", "mapping": {"type": "boolean", "store": True}}},
+    {"booleans": {"path_match": "*_bs", "mapping": {"type": "boolean", "store": True}}},
+    {
+        "date": {
+            "path_match": "*_dt",
+            "mapping": {
+                "type": "date",
+                "format": "date_optional_time||epoch_millis",
+                "store": True,
+            },
+        }
+    },
+    {
+        "date": {
+            "path_match": "*_dts",
+            "mapping": {
+                "type": "date",
+                "format": "date_optional_time||epoch_millis",
+                "store": True,
+            },
+        }
+    },
+]