howler-api 2.13.0.dev329__py3-none-any.whl

howler/common/classification.yml

@@ -0,0 +1,107 @@
+# This the default classification engine provided,
+#  it showcases all the different features of the classification engine
+#  while providing a useful configuration
+
+# Turn on/off classification enforcement. When this flag is off, this
+#  completely disables the classification engine, any documents added while
+#  the classification engine is off gets the default unrestricted value
+enforce: false
+dynamic_groups: false
+
+# List of Classification level:
+#   Graded list were a smaller number is less restricted then an higher number.
+levels:
+  # List of alternate names for the current marking
+  - aliases:
+      - UNRESTRICTED
+      - UNCLASSIFIED
+      - U
+    # Stylesheet applied in the UI for the different levels
+    css:
+      # Name of the color scheme used for display (default, primary, secondary, success, info, warning, error)
+      color: default
+    # Description of the classification level
+    description: Subject to standard copyright rules, TLP:WHITE information may be distributed without restriction.
+    # Interger value of the Classification level (higher is more classified)
+    lvl: 100
+    # Long name of the classification item
+    name: TLP:WHITE
+    # Short name of the classification item
+    short_name: TLP:W
+  - aliases: []
+    css:
+      color: success
+    description:
+      Recipients may share TLP:GREEN information with peers and partner organizations
+      within their sector or community, but not via publicly accessible channels. Information
+      in this category can be circulated widely within a particular community. TLP:GREEN
+      information may not be released outside of the community.
+    lvl: 110
+    name: TLP:GREEN
+    short_name: TLP:G
+  - aliases:
+      - RESTRICTED
+    css:
+      color: warning
+    description:
+      Recipients may only share TLP:AMBER information with members of their
+      own organization and with clients or customers who need to know the information
+      to protect themselves or prevent further harm.
+    lvl: 120
+    name: TLP:AMBER
+    short_name: TLP:A
+
+# List of required tokens:
+#   A user requesting access to an item must have all the
+#   required tokens the item has to gain access to it
+required:
+  - aliases: []
+    description: Produced using a commercial tool with limited distribution
+    name: COMMERCIAL
+    short_name: CMR
+    # The minimum classification level an item must have
+    #   for this token to be valid. (optional)
+    # require_lvl: 100
+
+# List of groups:
+#   A user requesting access to an item must be part of a least
+#   of one the group the item is part of to gain access
+groups:
+  - aliases: []
+    # This is a special flag that when set to true, if any groups are selected
+    #   in a classification. This group will automatically be selected too. (optional)
+    auto_select: true
+    description: Employees of CSE
+    name: CSE
+    short_name: CSE
+    # Assuming that this groups is the only group selected, this is the display name
+    #   that will be used in the classification (that values has to be in the aliases
+    #   of this group and only this group) (optional)
+    # solitary_display_name: ANY
+
+# List of subgroups:
+#   A user requesting access to an item must be part of a least
+#   of one the subgroup the item is part of to gain access
+subgroups:
+  - aliases: []
+    description: Member of Incident Response team
+    name: IR TEAM
+    short_name: IR
+  - aliases: []
+    description: Member of the Canadian Centre for Cyber Security
+    # This is a special flag that auto-select the corresponding group
+    #   when this subgroup is selected (optional)
+    require_group: CSE
+    name: CCCS
+    short_name: CCCS
+    # This is a special flag that makes sure that none other then the
+    #   corresponding group is selected when this subgroup is selected (optional)
+    # limited_to_group: CSE
+
+# Default restricted classification
+restricted: TLP:A//CMR
+
+# Default unrestricted classification:
+#   When no classification are provided or that the classification engine is
+#   disabled, this is the classification value each items will get
+unrestricted: TLP:W

howler/common/exceptions.py

@@ -0,0 +1,167 @@
+from inspect import getmembers, isfunction
+from sys import exc_info
+from traceback import format_tb
+from typing import Optional
+
+
+class HowlerException(Exception):
+    """Wrapper for all exceptions thrown in howler's code"""
+
+    message: str
+    cause: Optional[Exception]
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.cause = cause
+
+    def __repr__(self) -> str:
+        """String reproduction of the howler exception. Pass the message on"""
+        return self.message
+
+
+class InvalidClassification(HowlerException):
+    """Exception for Invalid Classification"""
+
+
+class InvalidDefinition(HowlerException):
+    """Exception for Invalid Definition"""
+
+
+class InvalidRangeException(HowlerException):
+    """Exception for Invalid Range"""
+
+
+class NonRecoverableError(HowlerException):
+    """Exception for an unrecoverable error"""
+
+
+class RecoverableError(HowlerException):
+    """Exception for a recoverable error"""
+
+
+class ConfigException(HowlerException):
+    """Exception thrown due to invalid configuration"""
+
+
+class ResourceExists(HowlerException):
+    """Exception thrown due to a pre-existing resource"""
+
+
+class VersionConflict(HowlerException):
+    """Exception thrown due to a version conflict"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause)
+
+
+class HowlerTypeError(HowlerException, TypeError):
+    """TypeError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else TypeError(message))
+
+
+class HowlerAttributeError(HowlerException, AttributeError):
+    """AttributeError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else AttributeError(message))
+
+
+class HowlerValueError(HowlerException, ValueError):
+    """ValueError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else ValueError(message))
+
+
+class HowlerNotImplementedError(HowlerException, NotImplementedError):
+    """NotImplementedError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else NotImplementedError(message))
+
+
+class HowlerKeyError(HowlerException, KeyError):
+    """KeyError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else KeyError(message))
+
+
+class HowlerRuntimeError(HowlerException, RuntimeError):
+    """RuntimeError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else RuntimeError(message))
+
+
+class NotFoundException(HowlerException):
+    """Exception thrown when a resource cannot be found"""
+
+
+class ForbiddenException(HowlerException):
+    """Exception thrown when a user is not permitted to perform an action"""
+
+
+class AccessDeniedException(HowlerException):
+    """Exception thrown when a resource cannot be accessed by a user"""
+
+
+class InvalidDataException(HowlerException):
+    """Exception thrown when user-provided data is invalid"""
+
+
+class AuthenticationException(HowlerException):
+    """Exception thrown when a user cannot be authenticated"""
+
+
+class Chain(object):
+    """This class can be used as a decorator to override the type of exceptions returned by a function"""
+
+    def __init__(self, exception):
+        self.exception = exception
+
+    def __call__(self, original):
+        """Execute a function and wrap any resulting exceptions"""
+
+        def wrapper(*args, **kwargs):
+            try:
+                return original(*args, **kwargs)
+            except Exception as e:
+                wrapped = self.exception(str(e), e)
+                raise wrapped.with_traceback(exc_info()[2])
+
+        wrapper.__name__ = original.__name__
+        wrapper.__doc__ = original.__doc__
+        wrapper.__dict__.update(original.__dict__)
+
+        return wrapper
+
+    def execute(self, func, *args, **kwargs):
+        """Execute a function and wrap any resulting exceptions"""
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            wrapped = self.exception(str(e), e)
+            raise wrapped.with_traceback(exc_info()[2])
+
+
+class ChainAll:
+    """This class can be used as a decorator to override the type of exceptions returned by every method of a class"""
+
+    def __init__(self, exception):
+        self.exception = Chain(exception)
+
+    def __call__(self, cls):
+        """We can use an instance of this class as a decorator."""
+        for method in getmembers(cls, predicate=isfunction):
+            setattr(cls, method[0], self.exception(method[1]))
+
+        return cls
+
+
+def get_stacktrace_info(ex: Exception) -> str:
+    """Get and format traceback information from a given exception"""
+    return "".join(format_tb(exc_info()[2]) + [": ".join((ex.__class__.__name__, str(ex)))])

howler/common/hexdump.py

@@ -0,0 +1,48 @@
+import binascii
+
+from howler.utils.chunk import chunk
+
+FILTER = b"".join([bytes([x]) if x in range(32, 127) else b"." for x in range(256)])
+
+
+def dump(binary: bytes, size: int = 2, sep: bytes = b" ") -> bytes:
+    "Convert a binary file into a hexadecimal representation"
+    hexstr = binascii.hexlify(binary)
+    return sep.join(chunk(hexstr, size))
+
+
+def hexdump(
+    binary: bytes,
+    length: int = 16,
+    indent: str = "",
+    indent_size: int = 0,
+    newline: str = "\n",
+    prefix_offset: int = 0,
+) -> str:
+    """Create a string buffer that shows the given data in hexdump format.
+
+    src -> source buffer
+    length = 16 -> number of bytes per line
+    indent = "" -> indentation before each lines
+    indent_size = 0 -> number of time to repeat that indentation
+    newline = "\n" -> chars used as newline char
+
+    Example of output:
+    00000000:  48 54 54 50 2F 31 2E 31 20 34 30 34 20 4E 6F 74  HTTP/1.1 404 Not
+    00000010:  20 46 6F 75 6E 64 0D 0A 43 6F 6E 74              Found..Cont
+    ...
+    """
+    generator = chunk(binary, length)
+    line_frmt = "%%s%%08X:  %%-%ss   %%s" % ((length * 3) - 1)
+
+    out = [
+        line_frmt
+        % (
+            indent * indent_size,
+            prefix_offset + (addr * length),
+            dump(d).decode(),
+            d.translate(FILTER).decode(),
+        )
+        for addr, d in enumerate(generator)
+    ]
+    return newline.join(out)

howler/common/iprange.py

@@ -0,0 +1,171 @@
+from math import log
+from socket import inet_aton
+from struct import pack, unpack
+
+from howler.common.exceptions import InvalidRangeException
+
+# If you are tempted to extend this module to add support for IPv6 (or some
+# similar invasive change) take a look at using PySubnetTree and extending it
+# to allow arbitrary ranges instead.
+
+
+def _convert(ip):
+    return unpack("!I", inet_aton(ip))[0]
+
+
+def _next(lower, upper):
+    size = 2
+    while lower + size - 1 <= upper and _valid(lower, lower + size - 1):
+        size *= 2
+
+    return int(size / 2)
+
+
+def _valid(lower, upper):
+    return lower & (upper - lower) == 0
+
+
+# noinspection PyPep8Naming
+class _dict(dict):  # noqa: N801
+    pass
+
+
+def ip_to_int(ip):
+    if isinstance(ip, int):
+        return ip
+
+    return _convert(ip)
+
+
+# noinspection PyTypeChecker
+class RangeTable(object):
+    """Efficient storage of IPv4 ranges and lookup of IPv4 addresses."""
+
+    def __init__(self):
+        self.clear()
+        self._trie = _dict()
+
+    def _add_cidr(self, lower, upper, value):
+        if not _valid(lower, upper):
+            # The public add_range method should ensure this never happens.
+            raise InvalidRangeException("invalid range: %d-%d" % (lower, upper))
+
+        size = upper - lower
+        points = 3 - int(log(size + 1, 256))
+
+        lower = self._to_path(lower)
+        upper = self._to_path(upper)
+
+        for x in range(lower[points], upper[points] + 1):
+            self._add_path(lower[:points] + (x,), value)
+
+    def _add_path(self, path, value):
+        trie = self._trie
+        for point in path[:-1]:
+            d = trie.get(point, _dict())
+            if not isinstance(d, _dict):
+                prev = d
+                d = _dict()
+                d.update({x: prev for x in range(256)})
+            trie[point] = d
+            trie = d
+        trie[path[-1]] = value
+
+    def _add_range(self, lower, upper, value):
+        while lower <= upper:
+            size = _next(lower, upper)
+            self._add_cidr(lower, lower + size - 1, value)
+            lower += size
+
+    def _follow_path(self, path):
+        entry = self._trie
+        while isinstance(entry, _dict):
+            entry = entry.get(path[0], None)
+            path = path[1:]
+        return entry
+
+    @staticmethod
+    def _to_path(integer):
+        return unpack("B" * 4, pack("!I", integer))
+
+    def __getitem__(self, key):
+        return self._follow_path(self._to_path(ip_to_int(key)))
+
+    def __setitem__(self, key, value):
+        if isinstance(key, int):
+            self._add_cidr(key, key, value)
+            return
+
+        span = key.split("-", 1)
+        if len(span) == 1:
+            span = key.split("/", 1)
+        if len(span) == 1:
+            span.append(span[0])
+
+        span[0] = _convert(span[0].strip())
+        if span[1].find(".") == -1:
+            mask = 2 ** (32 - int(span[1])) - 1
+            span[0] -= span[0] & mask
+            span[1] = span[0] | mask
+        else:
+            span[1] = _convert(span[1].strip())
+
+        self._add_range(span[0], span[1], value)
+
+    def add_range(self, lower, upper, value):
+        lower = ip_to_int(lower)
+        upper = ip_to_int(upper)
+
+        self._add_range(lower, upper, value)
+
+        while lower <= upper:
+            size = _next(lower, upper)
+            self._add_cidr(lower, lower + size - 1, value)
+            lower += size
+
+    def clear(self):
+        self._trie = _dict()  # pylint:disable=W0201
+
+    def dump(self):
+        from pprint import pformat
+
+        return pformat(self._trie)
+
+
+PRIVATE_NETWORKS = [
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16",
+]
+
+RESERVED_NETWORKS = [
+    "0.0.0.0/8",
+    "100.64.0.0/10",
+    "127.0.0.0/8",
+    "169.254.0.0/16",
+    "192.0.0.0/24",
+    "192.0.2.0/24",
+    "192.88.99.0/24",
+    "198.18.0.0/15",
+    "198.51.100.0/24",
+    "203.0.113.0/24",
+    "240.0.0.0/4",
+    "224.0.0.0/4",
+    "255.255.255.255/32",
+]
+
+_private_ips = RangeTable()
+for cidr in PRIVATE_NETWORKS:
+    _private_ips[cidr] = True
+
+_reserved_ips = RangeTable()
+for cidr in RESERVED_NETWORKS:
+    _reserved_ips[cidr] = True
+
+
+def is_ip_private(ip):
+    return _private_ips[ip] or False
+
+
+def is_ip_reserved(ip):
+    return _private_ips[ip] or _reserved_ips[ip] or False

howler/common/loader.py

@@ -0,0 +1,154 @@
+import logging
+import os
+from pathlib import Path
+from string import Template
+from typing import TYPE_CHECKING, Optional, Union
+
+import yaml
+
+from howler.odm.models.config import config
+
+if TYPE_CHECKING:
+    from howler.common.classification import Classification
+
+APP_NAME = os.environ.get("APP_NAME", "howler")
+APP_PREFIX = os.environ.get("APP_PREFIX", "hwl")
+USER_TYPES = {"admin", "user", "automation_basic", "automation_advanced"}
+
+
+def env_substitute(buffer):
+    """Replace environment variables in the buffer with their value.
+
+    Use the built in template expansion tool that expands environment variable style strings ${}
+    We set the idpattern to none so that $abc doesn't get replaced but ${abc} does.
+
+    Case insensitive.
+    Variables that are found in the buffer, but are not defined as environment variables are ignored.
+    """
+    return Template(buffer).safe_substitute(os.environ, idpattern=None, bracedidpattern="(?a:[_a-z][_a-z0-9]*)")
+
+
+_CLASSIFICATIONS: dict[Union[str, Path], "Classification"] = {}
+
+
+def get_classification(yml_config: Optional[str] = None):  # noqa: C901
+    "Get the classification from a given classification.yml file, caching results"
+    if yml_config in _CLASSIFICATIONS:
+        return _CLASSIFICATIONS[yml_config]
+
+    log = logging.getLogger(f"{APP_NAME}.common.loader")
+
+    if not yml_config:
+        yml_config_path = Path("/etc") / APP_NAME.replace("-dev", "") / "conf" / "classification.yml"
+        if yml_config_path.is_symlink():
+            log.info("%s is a symbolic link!", yml_config_path)
+            if str(yml_config_path.readlink()).startswith("..data"):
+                yml_config_path = yml_config_path.parent / yml_config_path.readlink()
+                log.info(
+                    "This symbolic link links to a configmap, handling accordingly. Reading from %s",
+                    yml_config_path,
+                )
+            else:
+                yml_config_path = Path(os.path.realpath(yml_config_path.readlink()))
+                log.info(
+                    "Reading from %s",
+                    yml_config_path,
+                )
+
+        if not yml_config_path.exists():
+            log.warning(f"{yml_config_path} does not exist!")
+            yml_config_path = Path("/etc") / APP_NAME.replace("-dev", "") / "classification.yml"
+            log.warning(f"Checking at {yml_config_path} instead.")
+    else:
+        yml_config_path = Path(yml_config)
+
+    log.debug("Loading classification definition from %s", yml_config_path)
+
+    classification_definition = None
+    # Load modifiers from the yaml config
+    if yml_config_path.exists():
+        with yml_config_path.open() as yml_fh:
+            yml_data = yaml.safe_load(yml_fh.read())
+            if yml_data:
+                classification_definition = yml_data
+
+    if classification_definition is None:
+        log.warning(
+            "Specified classification file does not exist or does not contain data!"
+            " Defaulting to default classification file."
+        )
+        default_file = Path(__file__).parent / "classification.yml"
+        if default_file.exists():
+            with default_file.open() as default_fh:
+                default_yml_data = yaml.safe_load(default_fh.read())
+                if default_yml_data:
+                    classification_definition = default_yml_data
+        else:
+            log.critical("%s was not accessible!", default_file)
+
+    from howler.common.classification import Classification, InvalidDefinition
+
+    if not classification_definition:
+        raise InvalidDefinition("Could not find any classification definition to load.")
+
+    _classification = Classification(classification_definition)
+
+    if yml_config:
+        _CLASSIFICATIONS[yml_config] = _classification
+
+    return _classification
+
+
+def get_lookups(lookup_folder: Optional[str] = None):
+    """Get lookups from the specified lookup folder"""
+    from howler.config import config
+
+    if not lookup_folder:
+        lookup_folder_path = Path("/etc/") / APP_NAME.replace("-dev", "") / "lookups"
+    else:
+        lookup_folder_path = Path(lookup_folder)
+
+    lookups = {}
+
+    if lookup_folder_path.exists():
+        for file in lookup_folder_path.iterdir():
+            with file.open("r") as f:
+                data = yaml.safe_load(f)
+                lookups[file.stem] = data
+    local_path = Path(__file__).parent.parent.parent / "static/mitre"
+    config_path = Path(config.ui.static_folder) if config.ui.static_folder else None
+    if local_path.exists():
+        mitre_path = local_path
+    elif config_path and (config_path / "mitre").exists():
+        mitre_path = config_path / "mitre"
+    else:
+        mitre_path = None
+
+    if mitre_path:
+        lookups["icons"] = sorted(set(f.stem for f in mitre_path.iterdir()))
+    else:
+        lookups["icons"] = []
+
+    lookups["roles"] = sorted(USER_TYPES)
+
+    return lookups
+
+
+# Lazy load the datastore
+_datastore = None
+
+
+def datastore(_config=None, archive_access=True):
+    """Get a datastore connection"""
+    global _datastore
+
+    from howler.datastore.howler_store import HowlerDatastore
+    from howler.datastore.store import ESStore
+
+    if not _config:
+        _config = config
+
+    if _datastore is None:
+        _datastore = HowlerDatastore(ESStore(config=config, archive_access=archive_access))
+
+    return _datastore