howler-api 3.0.0.dev374__py3-none-any.whl

howler/datastore/store.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import logging
+import os
+import re
+from os import environ
+from typing import Any, Optional, cast
+from urllib.parse import urlparse
+
+import elasticsearch
+import elasticsearch.client
+
+from howler.datastore.collection import ESCollection
+from howler.datastore.exceptions import DataStoreException
+from howler.odm.models.config import Config
+from howler.odm.models.config import config as _config
+
+TRANSPORT_TIMEOUT = int(environ.get("HWL_DATASTORE_TRANSPORT_TIMEOUT", "10"))
+
+
+class ESStore(object):
+    """Elasticsearch multi-index implementation of the ResultStore interface."""
+
+    DEFAULT_SORT = "id asc"
+    DATE_FORMAT = {
+        "NOW": "now",
+        "YEAR": "y",
+        "MONTH": "M",
+        "WEEK": "w",
+        "DAY": "d",
+        "HOUR": "h",
+        "MINUTE": "m",
+        "SECOND": "s",
+        "MILLISECOND": "ms",
+        "MICROSECOND": "micros",
+        "NANOSECOND": "nanos",
+        "SEPARATOR": "||",
+        "DATE_END": "Z",
+    }
+    DATEMATH_MAP = {
+        "NOW": "now",
+        "YEAR": "y",
+        "MONTH": "M",
+        "WEEK": "w",
+        "DAY": "d",
+        "HOUR": "h",
+        "MINUTE": "m",
+        "SECOND": "s",
+        "DATE_END": "Z||",
+    }
+    ID = "id"
+
+    def __init__(self, config: Optional[Config] = None, archive_access=True):
+        if not config:
+            config = _config
+
+        self._apikey: Optional[tuple[str, str]] = None
+        self._username: Optional[str] = None
+        self._password: Optional[str] = None
+        self._hosts = []
+
+        for host in config.datastore.hosts:
+            self._hosts.append(str(host))
+            if os.getenv(f"{host.name.upper()}_HOST_APIKEY_ID", None) is not None:
+                self._apikey = (
+                    os.environ[f"{host.name.upper()}_HOST_APIKEY_ID"],
+                    os.environ[f"{host.name.upper()}_HOST_APIKEY_SECRET"],
+                )
+            elif os.getenv(f"{host.name.upper()}_HOST_USERNAME") is not None:
+                self._username = os.environ[f"{host.name.upper()}_HOST_USERNAME"]
+                self._password = os.environ[f"{host.name.upper()}_HOST_PASSWORD"]
+
+        self._closed = False
+        self._collections: dict[str, ESCollection] = {}
+        self._models: dict[str, Any] = {}
+        self.validate = True
+
+        tracer = logging.getLogger("elasticsearch")
+        tracer.setLevel(logging.CRITICAL)
+
+        if self._apikey is not None:
+            self.client = elasticsearch.Elasticsearch(
+                hosts=self._hosts,  # type: ignore
+                api_key=self._apikey,
+                max_retries=0,
+                request_timeout=TRANSPORT_TIMEOUT,
+            )
+        elif self._username is not None and self._password is not None:
+            self.client = elasticsearch.Elasticsearch(
+                hosts=self._hosts,  # type: ignore
+                basic_auth=(self._username, self._password),
+                max_retries=0,
+                request_timeout=TRANSPORT_TIMEOUT,
+            )
+        else:
+            self.client = elasticsearch.Elasticsearch(
+                hosts=self._hosts,  # type: ignore
+                max_retries=0,
+                request_timeout=TRANSPORT_TIMEOUT,
+            )
+        self.eql = elasticsearch.client.EqlClient(self.client)
+        self.archive_access = archive_access
+        self.url_path = "elastic"
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, ex_type, exc_val, exc_tb):
+        self.close()
+
+    def __str__(self):
+        return "{0} - {1}".format(self.__class__.__name__, self._hosts)
+
+    def __getattr__(self, name) -> ESCollection:
+        if not self.validate:
+            return ESCollection(self, name, model_class=self._models[name], validate=self.validate)
+
+        if name not in self._collections:
+            self._collections[name] = ESCollection(self, name, model_class=self._models[name], validate=self.validate)
+
+        return self._collections[name]
+
+    @property
+    def now(self):
+        return self.DATE_FORMAT["NOW"]
+
+    @property
+    def ms(self):
+        return self.DATE_FORMAT["MILLISECOND"]
+
+    @property
+    def us(self):
+        return self.DATE_FORMAT["MICROSECOND"]
+
+    @property
+    def ns(self):
+        return self.DATE_FORMAT["NANOSECOND"]
+
+    @property
+    def year(self):
+        return self.DATE_FORMAT["YEAR"]
+
+    @property
+    def month(self):
+        return self.DATE_FORMAT["MONTH"]
+
+    @property
+    def week(self):
+        return self.DATE_FORMAT["WEEK"]
+
+    @property
+    def day(self):
+        return self.DATE_FORMAT["DAY"]
+
+    @property
+    def hour(self):
+        return self.DATE_FORMAT["HOUR"]
+
+    @property
+    def minute(self):
+        return self.DATE_FORMAT["MINUTE"]
+
+    @property
+    def second(self):
+        return self.DATE_FORMAT["SECOND"]
+
+    @property
+    def date_separator(self):
+        return self.DATE_FORMAT["SEPARATOR"]
+
+    def connection_reset(self):
+        self.client = elasticsearch.Elasticsearch(
+            hosts=self._hosts,  # type: ignore
+            api_key=self._apikey,
+            max_retries=0,
+            request_timeout=TRANSPORT_TIMEOUT,
+        )
+        self.eql = elasticsearch.client.EqlClient(self.client)
+
+    def close(self):
+        self._closed = True
+        # Flatten the client object so that attempts to access without reconnecting errors hard
+        # But 'cast' it so that mypy and other linters don't think that its normal for client to be None
+        self.client = cast(elasticsearch.Elasticsearch, None)
+        self.eql = cast(elasticsearch.client.EqlClient, None)
+
+    def get_hosts(self, safe=False):
+        if not safe:
+            return self._hosts
+        else:
+            out = []
+            for h in self._hosts:
+                parsed = urlparse(h)
+                out.append(parsed.hostname or parsed.path)
+            return out
+
+    def get_models(self):
+        return self._models
+
+    def is_closed(self):
+        return self._closed
+
+    def ping(self):
+        return self.client.ping()
+
+    def register(self, name: str, model_class=None):
+        name_match = re.match(r"[a-z0-9_]*", name)
+        if not name_match or name_match.string != name:
+            raise DataStoreException(
+                "Invalid characters in model name. " "You can only use lower case letters, numbers and underscores."
+            )
+
+        self._models[name] = model_class
+
+    def to_pydatemath(self, value):
+        replace_list = [
+            (self.now, self.DATEMATH_MAP["NOW"]),
+            (self.year, self.DATEMATH_MAP["YEAR"]),
+            (self.month, self.DATEMATH_MAP["MONTH"]),
+            (self.week, self.DATEMATH_MAP["WEEK"]),
+            (self.day, self.DATEMATH_MAP["DAY"]),
+            (self.hour, self.DATEMATH_MAP["HOUR"]),
+            (self.minute, self.DATEMATH_MAP["MINUTE"]),
+            (self.second, self.DATEMATH_MAP["SECOND"]),
+            (self.DATE_FORMAT["DATE_END"], self.DATEMATH_MAP["DATE_END"]),
+        ]
+
+        for x in replace_list:
+            value = value.replace(*x)
+
+        return value

howler/datastore/support/build.py

@@ -0,0 +1,215 @@
+from typing import Union
+
+from howler.common.exceptions import HowlerNotImplementedError, HowlerValueError
+from howler.common.logging import get_logger
+from howler.datastore.constants import ANALYZER_MAPPING, NORMALIZER_MAPPING, TYPE_MAPPING
+from howler.odm import (
+    Any,
+    Boolean,
+    Classification,
+    Compound,
+    Date,
+    FlattenedObject,
+    Float,
+    Integer,
+    Json,
+    Keyword,
+    List,
+    Long,
+    Mapping,
+    Optional,
+    Text,
+)
+
+logger = get_logger(__file__)
+
+
+def build_mapping(field_data, prefix=None, allow_refuse_implicit=True):
+    """The mapping for Elasticsearch based on a python model object."""
+    prefix = prefix or []
+    mappings = {}
+    dynamic = []
+
+    def set_mapping(temp_field, body):
+        body["index"] = temp_field.index
+        if body.get("type", "text") != "text":
+            body["doc_values"] = temp_field.index
+        if temp_field.copyto:
+            if len(field.copyto) > 1:
+                logger.warning("copyto field larger than 1, only using first entry")
+            body["copy_to"] = temp_field.copyto[0]
+
+        return body
+
+    # Fill in the sections
+    for field in field_data:
+        path = prefix + ([field.name] if field.name else [])
+        name = ".".join(path)
+
+        if isinstance(field, Classification):
+            mappings[name.strip(".")] = set_mapping(field, {"type": TYPE_MAPPING[field.__class__.__name__]})
+            if "." not in name:
+                mappings.update(
+                    {
+                        "__access_lvl__": {"type": "integer", "index": True},
+                        "__access_req__": {"type": "keyword", "index": True},
+                        "__access_grp1__": {"type": "keyword", "index": True},
+                        "__access_grp2__": {"type": "keyword", "index": True},
+                    }
+                )
+
+        elif isinstance(field, (Boolean, Integer, Float, Text, Long)):
+            mappings[name.strip(".")] = set_mapping(field, {"type": TYPE_MAPPING[field.__class__.__name__]})
+
+        elif field.__class__ in ANALYZER_MAPPING:
+            mappings[name.strip(".")] = set_mapping(
+                field,
+                {
+                    "type": TYPE_MAPPING[field.__class__.__name__],
+                    "analyzer": ANALYZER_MAPPING[field.__class__],
+                },
+            )
+
+        elif isinstance(field, (Json, Keyword)):
+            es_data_type = TYPE_MAPPING[field.__class__.__name__]
+            data: dict[str, Union[str, int]] = {"type": es_data_type}
+            if es_data_type == "keyword":
+                data["ignore_above"] = 8191  # The maximum always safe value in elasticsearch
+            if field.__class__ in NORMALIZER_MAPPING:
+                data["normalizer"] = NORMALIZER_MAPPING[field.__class__]  # type: ignore
+            mappings[name.strip(".")] = set_mapping(field, data)
+
+        elif isinstance(field, Date):
+            mappings[name.strip(".")] = set_mapping(
+                field,
+                {
+                    "type": TYPE_MAPPING[field.__class__.__name__],
+                    "format": "date_optional_time||epoch_millis",
+                },
+            )
+
+        elif isinstance(field, FlattenedObject):
+            if not field.index or isinstance(field.child_type, Any):
+                mappings[name.strip(".")] = {"type": "object", "enabled": False}
+            else:
+                dynamic.extend(
+                    build_templates(
+                        f"{name}.*",
+                        field.child_type,
+                        nested_template=True,
+                        index=field.index,
+                    )
+                )
+
+        elif isinstance(field, List):
+            temp_mappings, temp_dynamic = build_mapping([field.child_type], prefix=path, allow_refuse_implicit=False)
+            mappings.update(temp_mappings)
+            dynamic.extend(temp_dynamic)
+
+        elif isinstance(field, Optional):
+            temp_mappings, temp_dynamic = build_mapping([field.child_type], prefix=prefix, allow_refuse_implicit=False)
+            mappings.update(temp_mappings)
+            dynamic.extend(temp_dynamic)
+
+        elif isinstance(field, Compound):
+            temp_mappings, temp_dynamic = build_mapping(
+                field.fields().values(), prefix=path, allow_refuse_implicit=False
+            )
+            mappings.update(temp_mappings)
+            dynamic.extend(temp_dynamic)
+
+        elif isinstance(field, Mapping):
+            if not field.index or isinstance(field.child_type, Any):
+                mappings[name.strip(".")] = {"type": "object", "enabled": False}
+            else:
+                dynamic.extend(build_templates(f"{name}.*", field.child_type, index=field.index))
+
+        elif isinstance(field, Any):
+            if field.index:
+                raise HowlerValueError(f"Any may not be indexed: {name}")
+
+            mappings[name.strip(".")] = {
+                "type": "keyword",
+                "index": False,
+                "doc_values": False,
+            }
+
+        else:
+            raise HowlerNotImplementedError(f"Unknown type for elasticsearch schema: {field.__class__}")
+
+    # The final template must match everything and disable indexing
+    # this effectively disables dynamic indexing EXCEPT for the templates
+    # we have defined
+    if not dynamic and allow_refuse_implicit:
+        # We cannot use the dynamic type matching if others are in play because they conflict with each other
+        # TODO: Find a way to make them work together.
+        dynamic.append(
+            {
+                "refuse_all_implicit_mappings": {
+                    "match": "*",
+                    "mapping": {
+                        "index": False,
+                        "ignore_malformed": True,
+                    },
+                }
+            }
+        )
+
+    return mappings, dynamic
+
+
+def build_templates(name, field, nested_template=False, index=True) -> list:
+    if isinstance(field, (Keyword, Boolean, Integer, Float, Text, Json)):
+        if nested_template:
+            main_template = {"match": f"{name}", "mapping": {"type": "nested"}}
+
+            return [{f"nested_{name}": main_template}]
+        else:
+            field_template = {
+                "path_match": name,
+                "mapping": {
+                    "type": TYPE_MAPPING[field.__class__.__name__],
+                    "index": field.index,
+                },
+            }
+
+            if field.copyto:
+                if len(field.copyto) > 1:
+                    logger.warning("copyto field larger than 1, only using first entry")
+                field_template["mapping"]["copy_to"] = field.copyto[0]
+
+            return [{f"{name}_tpl": field_template}]
+
+    elif isinstance(field, Any) or not index:
+        field_template = {
+            "path_match": name,
+            "mapping": {"type": "keyword", "index": False},
+        }
+
+        if field.index:
+            raise HowlerValueError(f"Mapping to Any may not be indexed: {name}")
+        return [{f"{name}_tpl": field_template}]
+
+    elif isinstance(field, (Mapping, List)):
+        temp_name = name
+        if field.name:
+            temp_name = f"{name}.{field.name}"
+        return build_templates(temp_name, field.child_type, nested_template=True)
+
+    elif isinstance(field, Compound):
+        temp_name = name
+        if field.name:
+            temp_name = f"{name}.{field.name}"
+
+        out = []
+        for sub_name, sub_field in field.fields().items():
+            sub_name = f"{temp_name}.{sub_name}"
+            out.extend(build_templates(sub_name, sub_field))
+
+        return out
+
+    elif isinstance(field, Optional):
+        return build_templates(name, field.child_type, nested_template=nested_template)
+
+    else:
+        raise HowlerNotImplementedError(f"Unknown type for elasticsearch dynamic mapping: {field.__class__}")

howler/datastore/support/schemas.py

@@ -0,0 +1,90 @@
+default_index = {
+    "settings": {
+        "analysis": {
+            "filter": {
+                "text_ws_dsplit": {
+                    "type": "pattern_replace",
+                    "pattern": r"(\.)",
+                    "replacement": " ",
+                }
+            },
+            "analyzer": {
+                "string_ci": {
+                    "type": "custom",
+                    "tokenizer": "keyword",
+                    "filter": ["lowercase"],
+                },
+                "text_fuzzy": {
+                    "type": "pattern",
+                    "pattern": r"\s*:\s*",
+                    "lowercase": False,
+                },
+                "text_whitespace": {"type": "whitespace"},
+                "text_ws_dsplit": {
+                    "type": "custom",
+                    "tokenizer": "whitespace",
+                    "filters": ["text_ws_dsplit"],
+                },
+            },
+            "normalizer": {
+                "lowercase_normalizer": {
+                    "type": "custom",
+                    "char_filter": [],
+                    "filter": ["lowercase"],
+                }
+            },
+        }
+    },
+    "mappings": {},
+}
+
+default_mapping = {
+    "dynamic": True,
+    "properties": {
+        "__text__": {"type": "text"},
+    },
+}
+
+default_dynamic_strings = {
+    "strings_as_keywords": {
+        "match_mapping_type": "string",
+        "mapping": {"type": "keyword", "ignore_above": 8191},
+    }
+}
+
+default_dynamic_templates = [
+    {"int": {"path_match": "*_i", "mapping": {"type": "integer", "store": True}}},
+    {"ints": {"path_match": "*_is", "mapping": {"type": "integer", "store": True}}},
+    {"long": {"path_match": "*_l", "mapping": {"type": "long", "store": True}}},
+    {"longs": {"path_match": "*_ls", "mapping": {"type": "long", "store": True}}},
+    {"double": {"path_match": "*_d", "mapping": {"type": "float", "store": True}}},
+    {"doubles": {"path_match": "*_ds", "mapping": {"type": "float", "store": True}}},
+    {"float": {"path_match": "*_f", "mapping": {"type": "float", "store": True}}},
+    {"floats": {"path_match": "*_fs", "mapping": {"type": "float", "store": True}}},
+    {"string": {"path_match": "*_s", "mapping": {"type": "keyword", "store": True}}},
+    {"strings": {"path_match": "*_ss", "mapping": {"type": "keyword", "store": True}}},
+    {"text": {"path_match": "*_t", "mapping": {"type": "text", "store": True}}},
+    {"texts": {"path_match": "*_ts", "mapping": {"type": "text", "store": True}}},
+    {"boolean": {"path_match": "*_b", "mapping": {"type": "boolean", "store": True}}},
+    {"booleans": {"path_match": "*_bs", "mapping": {"type": "boolean", "store": True}}},
+    {
+        "date": {
+            "path_match": "*_dt",
+            "mapping": {
+                "type": "date",
+                "format": "date_optional_time||epoch_millis",
+                "store": True,
+            },
+        }
+    },
+    {
+        "date": {
+            "path_match": "*_dts",
+            "mapping": {
+                "type": "date",
+                "format": "date_optional_time||epoch_millis",
+                "store": True,
+            },
+        }
+    },
+]

howler/datastore/types.py

@@ -0,0 +1,22 @@
+from typing import Any, TypedDict, Union
+
+from howler.odm import Model
+from howler.odm.models.hit import Hit
+
+
+class SearchResult(TypedDict):
+    """Search Results for an Elastic search query.
+
+    TypedDict's are used in the same way as a normal Dict, but provides typing for InteliSense
+    """
+
+    offset: int
+    rows: int
+    total: int
+    items: list[Union[Model, dict[str, Any]]]
+
+
+class HitSearchResult(SearchResult):
+    """Hit specific typing for Elastic search query"""
+
+    items: list[Union[Hit, dict[str, Any]]]  # type: ignore[misc]

howler/error.py

@@ -0,0 +1,91 @@
+from sys import exc_info
+from traceback import format_tb
+
+from flask import Blueprint, request
+from werkzeug.exceptions import BadRequest, Forbidden, Unauthorized
+
+from howler.api import bad_request, forbidden, internal_error, not_found, unauthorized
+from howler.common.exceptions import AccessDeniedException, AuthenticationException
+from howler.common.logging import get_logger, log_with_traceback
+from howler.common.logging.audit import AUDIT
+from howler.config import config
+
+errors = Blueprint("errors", __name__)
+
+logger = get_logger(__file__)
+
+
+######################################
+# Custom Error page
+@errors.app_errorhandler(400)
+def handle_400(e):
+    """Handle bad request errors"""
+    if isinstance(e, BadRequest):
+        error_message = "No data block provided or data block not in JSON format.'"
+    else:
+        error_message = str(e)
+    return bad_request(err=error_message)
+
+
+@errors.app_errorhandler(401)
+def handle_401(e):
+    """Handle unauthorized errors"""
+    if isinstance(e, Unauthorized):
+        msg = e.description
+    else:
+        msg = str(e)
+
+    data = {
+        "oauth_providers": [name for name in config.auth.oauth.providers.keys()],
+        "allow_userpass_login": config.auth.internal.enabled,
+    }
+    res = unauthorized(data, err=msg)
+    res.set_cookie("XSRF-TOKEN", "", max_age=0, secure=True, httponly=True, samesite="Strict")
+    return res
+
+
+@errors.app_errorhandler(403)
+def handle_403(e):
+    """Handle bad forbidden errors"""
+    if isinstance(e, Forbidden):
+        error_message = e.description
+    else:
+        error_message = str(e)
+
+    trace = exc_info()[2]
+    if AUDIT:
+        uname = "(None)"
+        ip = request.remote_addr
+
+        log_with_traceback(trace, f"Access Denied. (U:{uname} - IP:{ip}) [{error_message}]", audit=True)
+
+    config_block = {
+        "auth": {
+            "allow_apikeys": config.auth.allow_apikeys,
+        }
+    }
+    return forbidden(config_block, err=f"Access Denied ({request.path}) [{error_message}]")
+
+
+@errors.app_errorhandler(404)
+def handle_404(_):
+    """Handle not found errors"""
+    return not_found(err=f"Api does not exist ({request.path})")
+
+
+@errors.app_errorhandler(500)
+def handle_500(e):
+    """Handle internal server errors"""
+    if isinstance(e.original_exception, AccessDeniedException):
+        return handle_403(e.original_exception)
+
+    if isinstance(e.original_exception, AuthenticationException):
+        return handle_401(e.original_exception)
+
+    oe = e.original_exception or e
+
+    trace = exc_info()[2]
+    log_with_traceback(trace, "Exception", is_exception=True)
+
+    message = "".join(["\n"] + format_tb(exc_info()[2]) + ["%s: %s\n" % (oe.__class__.__name__, str(oe))]).rstrip("\n")
+    return internal_error(err=message)