howler-api 3.0.0.dev374__py3-none-any.whl

howler/common/classification.yml

@@ -0,0 +1,107 @@
+# This the default classification engine provided,
+#  it showcases all the different features of the classification engine
+#  while providing a useful configuration
+
+# Turn on/off classification enforcement. When this flag is off, this
+#  completely disables the classification engine, any documents added while
+#  the classification engine is off gets the default unrestricted value
+enforce: false
+dynamic_groups: false
+
+# List of Classification level:
+#   Graded list were a smaller number is less restricted then an higher number.
+levels:
+  # List of alternate names for the current marking
+  - aliases:
+      - UNRESTRICTED
+      - UNCLASSIFIED
+      - U
+    # Stylesheet applied in the UI for the different levels
+    css:
+      # Name of the color scheme used for display (default, primary, secondary, success, info, warning, error)
+      color: default
+    # Description of the classification level
+    description: Subject to standard copyright rules, TLP:WHITE information may be distributed without restriction.
+    # Interger value of the Classification level (higher is more classified)
+    lvl: 100
+    # Long name of the classification item
+    name: TLP:WHITE
+    # Short name of the classification item
+    short_name: TLP:W
+  - aliases: []
+    css:
+      color: success
+    description:
+      Recipients may share TLP:GREEN information with peers and partner organizations
+      within their sector or community, but not via publicly accessible channels. Information
+      in this category can be circulated widely within a particular community. TLP:GREEN
+      information may not be released outside of the community.
+    lvl: 110
+    name: TLP:GREEN
+    short_name: TLP:G
+  - aliases:
+      - RESTRICTED
+    css:
+      color: warning
+    description:
+      Recipients may only share TLP:AMBER information with members of their
+      own organization and with clients or customers who need to know the information
+      to protect themselves or prevent further harm.
+    lvl: 120
+    name: TLP:AMBER
+    short_name: TLP:A
+
+# List of required tokens:
+#   A user requesting access to an item must have all the
+#   required tokens the item has to gain access to it
+required:
+  - aliases: []
+    description: Produced using a commercial tool with limited distribution
+    name: COMMERCIAL
+    short_name: CMR
+    # The minimum classification level an item must have
+    #   for this token to be valid. (optional)
+    # require_lvl: 100
+
+# List of groups:
+#   A user requesting access to an item must be part of a least
+#   of one the group the item is part of to gain access
+groups:
+  - aliases: []
+    # This is a special flag that when set to true, if any groups are selected
+    #   in a classification. This group will automatically be selected too. (optional)
+    auto_select: true
+    description: Employees of CSE
+    name: CSE
+    short_name: CSE
+    # Assuming that this groups is the only group selected, this is the display name
+    #   that will be used in the classification (that values has to be in the aliases
+    #   of this group and only this group) (optional)
+    # solitary_display_name: ANY
+
+# List of subgroups:
+#   A user requesting access to an item must be part of a least
+#   of one the subgroup the item is part of to gain access
+subgroups:
+  - aliases: []
+    description: Member of Incident Response team
+    name: IR TEAM
+    short_name: IR
+  - aliases: []
+    description: Member of the Canadian Centre for Cyber Security
+    # This is a special flag that auto-select the corresponding group
+    #   when this subgroup is selected (optional)
+    require_group: CSE
+    name: CCCS
+    short_name: CCCS
+    # This is a special flag that makes sure that none other then the
+    #   corresponding group is selected when this subgroup is selected (optional)
+    # limited_to_group: CSE
+
+# Default restricted classification
+restricted: TLP:A//CMR
+
+# Default unrestricted classification:
+#   When no classification are provided or that the classification engine is
+#   disabled, this is the classification value each items will get
+unrestricted: TLP:W

howler/common/exceptions.py

@@ -0,0 +1,167 @@
+from inspect import getmembers, isfunction
+from sys import exc_info
+from traceback import format_tb
+from typing import Optional
+
+
+class HowlerException(Exception):
+    """Wrapper for all exceptions thrown in howler's code"""
+
+    message: str
+    cause: Optional[Exception]
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.cause = cause
+
+    def __repr__(self) -> str:
+        """String reproduction of the howler exception. Pass the message on"""
+        return self.message
+
+
+class InvalidClassification(HowlerException):
+    """Exception for Invalid Classification"""
+
+
+class InvalidDefinition(HowlerException):
+    """Exception for Invalid Definition"""
+
+
+class InvalidRangeException(HowlerException):
+    """Exception for Invalid Range"""
+
+
+class NonRecoverableError(HowlerException):
+    """Exception for an unrecoverable error"""
+
+
+class RecoverableError(HowlerException):
+    """Exception for a recoverable error"""
+
+
+class ConfigException(HowlerException):
+    """Exception thrown due to invalid configuration"""
+
+
+class ResourceExists(HowlerException):
+    """Exception thrown due to a pre-existing resource"""
+
+
+class VersionConflict(HowlerException):
+    """Exception thrown due to a version conflict"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause)
+
+
+class HowlerTypeError(HowlerException, TypeError):
+    """TypeError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else TypeError(message))
+
+
+class HowlerAttributeError(HowlerException, AttributeError):
+    """AttributeError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else AttributeError(message))
+
+
+class HowlerValueError(HowlerException, ValueError):
+    """ValueError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else ValueError(message))
+
+
+class HowlerNotImplementedError(HowlerException, NotImplementedError):
+    """NotImplementedError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else NotImplementedError(message))
+
+
+class HowlerKeyError(HowlerException, KeyError):
+    """KeyError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else KeyError(message))
+
+
+class HowlerRuntimeError(HowlerException, RuntimeError):
+    """RuntimeError child specifically for exceptions thrown by us"""
+
+    def __init__(self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        HowlerException.__init__(self, message, cause if cause is not None else RuntimeError(message))
+
+
+class NotFoundException(HowlerException):
+    """Exception thrown when a resource cannot be found"""
+
+
+class ForbiddenException(HowlerException):
+    """Exception thrown when a user is not permitted to perform an action"""
+
+
+class AccessDeniedException(HowlerException):
+    """Exception thrown when a resource cannot be accessed by a user"""
+
+
+class InvalidDataException(HowlerException):
+    """Exception thrown when user-provided data is invalid"""
+
+
+class AuthenticationException(HowlerException):
+    """Exception thrown when a user cannot be authenticated"""
+
+
+class Chain(object):
+    """This class can be used as a decorator to override the type of exceptions returned by a function"""
+
+    def __init__(self, exception):
+        self.exception = exception
+
+    def __call__(self, original):
+        """Execute a function and wrap any resulting exceptions"""
+
+        def wrapper(*args, **kwargs):
+            try:
+                return original(*args, **kwargs)
+            except Exception as e:
+                wrapped = self.exception(str(e), e)
+                raise wrapped.with_traceback(exc_info()[2])
+
+        wrapper.__name__ = original.__name__
+        wrapper.__doc__ = original.__doc__
+        wrapper.__dict__.update(original.__dict__)
+
+        return wrapper
+
+    def execute(self, func, *args, **kwargs):
+        """Execute a function and wrap any resulting exceptions"""
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            wrapped = self.exception(str(e), e)
+            raise wrapped.with_traceback(exc_info()[2])
+
+
+class ChainAll:
+    """This class can be used as a decorator to override the type of exceptions returned by every method of a class"""
+
+    def __init__(self, exception):
+        self.exception = Chain(exception)
+
+    def __call__(self, cls):
+        """We can use an instance of this class as a decorator."""
+        for method in getmembers(cls, predicate=isfunction):
+            setattr(cls, method[0], self.exception(method[1]))
+
+        return cls
+
+
+def get_stacktrace_info(ex: Exception) -> str:
+    """Get and format traceback information from a given exception"""
+    return "".join(format_tb(exc_info()[2]) + [": ".join((ex.__class__.__name__, str(ex)))])

howler/common/loader.py

@@ -0,0 +1,154 @@
+import logging
+import os
+from pathlib import Path
+from string import Template
+from typing import TYPE_CHECKING, Optional, Union
+
+import yaml
+
+from howler.odm.models.config import config
+
+if TYPE_CHECKING:
+    from howler.common.classification import Classification
+
+APP_NAME = os.environ.get("APP_NAME", "howler")
+APP_PREFIX = os.environ.get("APP_PREFIX", "hwl")
+USER_TYPES = {"admin", "user", "automation_basic", "automation_advanced"}
+
+
+def env_substitute(buffer):
+    """Replace environment variables in the buffer with their value.
+
+    Use the built in template expansion tool that expands environment variable style strings ${}
+    We set the idpattern to none so that $abc doesn't get replaced but ${abc} does.
+
+    Case insensitive.
+    Variables that are found in the buffer, but are not defined as environment variables are ignored.
+    """
+    return Template(buffer).safe_substitute(os.environ, idpattern=None, bracedidpattern="(?a:[_a-z][_a-z0-9]*)")
+
+
+_CLASSIFICATIONS: dict[Union[str, Path], "Classification"] = {}
+
+
+def get_classification(yml_config: Optional[str] = None):  # noqa: C901
+    "Get the classification from a given classification.yml file, caching results"
+    if yml_config in _CLASSIFICATIONS:
+        return _CLASSIFICATIONS[yml_config]
+
+    log = logging.getLogger(f"{APP_NAME}.common.loader")
+
+    if not yml_config:
+        yml_config_path = Path("/etc") / APP_NAME.replace("-dev", "") / "conf" / "classification.yml"
+        if yml_config_path.is_symlink():
+            log.info("%s is a symbolic link!", yml_config_path)
+            if str(yml_config_path.readlink()).startswith("..data"):
+                yml_config_path = yml_config_path.parent / yml_config_path.readlink()
+                log.info(
+                    "This symbolic link links to a configmap, handling accordingly. Reading from %s",
+                    yml_config_path,
+                )
+            else:
+                yml_config_path = Path(os.path.realpath(yml_config_path.readlink()))
+                log.info(
+                    "Reading from %s",
+                    yml_config_path,
+                )
+
+        if not yml_config_path.exists():
+            log.warning(f"{yml_config_path} does not exist!")
+            yml_config_path = Path("/etc") / APP_NAME.replace("-dev", "") / "classification.yml"
+            log.warning(f"Checking at {yml_config_path} instead.")
+    else:
+        yml_config_path = Path(yml_config)
+
+    log.debug("Loading classification definition from %s", yml_config_path)
+
+    classification_definition = None
+    # Load modifiers from the yaml config
+    if yml_config_path.exists():
+        with yml_config_path.open() as yml_fh:
+            yml_data = yaml.safe_load(yml_fh.read())
+            if yml_data:
+                classification_definition = yml_data
+
+    if classification_definition is None:
+        log.warning(
+            "Specified classification file does not exist or does not contain data!"
+            " Defaulting to default classification file."
+        )
+        default_file = Path(__file__).parent / "classification.yml"
+        if default_file.exists():
+            with default_file.open() as default_fh:
+                default_yml_data = yaml.safe_load(default_fh.read())
+                if default_yml_data:
+                    classification_definition = default_yml_data
+        else:
+            log.critical("%s was not accessible!", default_file)
+
+    from howler.common.classification import Classification, InvalidDefinition
+
+    if not classification_definition:
+        raise InvalidDefinition("Could not find any classification definition to load.")
+
+    _classification = Classification(classification_definition)
+
+    if yml_config:
+        _CLASSIFICATIONS[yml_config] = _classification
+
+    return _classification
+
+
+def get_lookups(lookup_folder: Optional[str] = None):
+    """Get lookups from the specified lookup folder"""
+    from howler.config import config
+
+    if not lookup_folder:
+        lookup_folder_path = Path("/etc/") / APP_NAME.replace("-dev", "") / "lookups"
+    else:
+        lookup_folder_path = Path(lookup_folder)
+
+    lookups = {}
+
+    if lookup_folder_path.exists():
+        for file in lookup_folder_path.iterdir():
+            with file.open("r") as f:
+                data = yaml.safe_load(f)
+                lookups[file.stem] = data
+    local_path = Path(__file__).parent.parent.parent / "static/mitre"
+    config_path = Path(config.ui.static_folder) if config.ui.static_folder else None
+    if local_path.exists():
+        mitre_path = local_path
+    elif config_path and (config_path / "mitre").exists():
+        mitre_path = config_path / "mitre"
+    else:
+        mitre_path = None
+
+    if mitre_path:
+        lookups["icons"] = sorted(set(f.stem for f in mitre_path.iterdir()))
+    else:
+        lookups["icons"] = []
+
+    lookups["roles"] = sorted(USER_TYPES)
+
+    return lookups
+
+
+# Lazy load the datastore
+_datastore = None
+
+
+def datastore(_config=None, archive_access=True):
+    """Get a datastore connection"""
+    global _datastore
+
+    from howler.datastore.howler_store import HowlerDatastore
+    from howler.datastore.store import ESStore
+
+    if not _config:
+        _config = config
+
+    if _datastore is None:
+        _datastore = HowlerDatastore(ESStore(config=config, archive_access=archive_access))
+
+    return _datastore

howler/common/logging/__init__.py

@@ -0,0 +1,241 @@
+import json
+import logging
+import logging.handlers
+import os
+import re
+from traceback import format_exception
+from typing import TYPE_CHECKING, Optional, Union
+
+from flask import request
+from typing_extensions import override  # type: ignore
+
+from howler.common import loader
+from howler.common.logging.format import (
+    HWL_DATE_FORMAT,
+    HWL_JSON_FORMAT,
+    HWL_LOG_FORMAT,
+    HWL_SYSLOG_FORMAT,
+)
+
+if TYPE_CHECKING:
+    from howler.odm.models.config import Config
+
+LOG_LEVEL_MAP = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+    "DISABLED": 60,
+}
+
+DEBUG = False
+
+
+class JsonFormatter(logging.Formatter):
+    """logging Formatter to output in JSON"""
+
+    @override
+    def formatMessage(self, record):
+        if record.exc_info:
+            record.exc_text = self.formatException(record.exc_info)
+            record.exc_info = None
+
+        if record.exc_text:
+            record.message += "\n" + record.exc_text
+            record.exc_text = None
+
+        record.message = json.dumps(record.message)
+        return self._style.format(record)
+
+    @override
+    def formatException(self, exc_info):
+        return "".join(format_exception(*exc_info))
+
+
+def init_log_to_file(logger: logging.Logger, log_level: int, name: str, config: "Config"):
+    """Initialize file-based logging"""
+    if not os.path.isdir(config.logging.log_directory):
+        logger.warning(
+            "Log directory does not exist. Will try to create %s",
+            config.logging.log_directory,
+        )
+        os.makedirs(config.logging.log_directory)
+
+    if log_level <= logging.DEBUG:
+        dbg_file_handler = logging.handlers.RotatingFileHandler(
+            os.path.join(config.logging.log_directory, f"{name}.dbg"),
+            maxBytes=10485760,
+            backupCount=5,
+        )
+        dbg_file_handler.setLevel(logging.DEBUG)
+        if config.logging.log_as_json:
+            dbg_file_handler.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            dbg_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(dbg_file_handler)
+
+    if log_level <= logging.INFO:
+        op_file_handler = logging.handlers.RotatingFileHandler(
+            os.path.join(config.logging.log_directory, f"{name}.log"),
+            maxBytes=10485760,
+            backupCount=5,
+        )
+        op_file_handler.setLevel(logging.INFO)
+        if config.logging.log_as_json:
+            op_file_handler.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            op_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(op_file_handler)
+
+    if log_level <= logging.ERROR:
+        err_file_handler = logging.handlers.RotatingFileHandler(
+            os.path.join(config.logging.log_directory, f"{name}.err"),
+            maxBytes=10485760,
+            backupCount=5,
+        )
+        err_file_handler.setLevel(logging.ERROR)
+        if config.logging.log_as_json:
+            err_file_handler.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            err_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        err_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(err_file_handler)
+
+
+def init_logging(name: str, log_level: Optional[int] = None):
+    """Initialize the logger"""
+    from howler.config import config
+
+    logger = logging.getLogger(loader.APP_NAME)
+
+    # Test if we've initialized the log handler already.
+    if len(logger.handlers) != 0:
+        return logger.getChild(name)
+
+    if name.startswith(f"{loader.APP_NAME}."):
+        name = name[len(loader.APP_NAME) + 1 :]
+
+    config.logging.log_to_console = config.logging.log_to_console or config.ui.debug
+
+    if log_level is None:
+        log_level = LOG_LEVEL_MAP[config.logging.log_level]
+
+    logging.root.setLevel(logging.CRITICAL)
+    logger.setLevel(log_level)
+
+    if config.logging.log_level == "DISABLED":
+        # While log_level is set to disable, we will not create any handlers
+        return logger.getChild(name)
+
+    if config.logging.log_to_file:
+        init_log_to_file(logger, log_level, name, config)
+
+    if config.logging.log_to_console:
+        console = logging.StreamHandler()
+        if config.logging.log_as_json:
+            console.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            console.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(console)
+
+    if config.logging.log_to_syslog and config.logging.syslog_host and config.logging.syslog_port:
+        syslog_handler = logging.handlers.SysLogHandler(
+            address=(config.logging.syslog_host, config.logging.syslog_port)
+        )
+        syslog_handler.formatter = logging.Formatter(HWL_SYSLOG_FORMAT)
+        logger.addHandler(syslog_handler)
+
+    logger.debug("Logger ready!")
+    return logger.getChild(name)
+
+
+def get_logger(name: str = "default") -> logging.Logger:
+    """Get a logger with a useful name given a filename"""
+    name = re.sub(r".+(howler|test)/", "", name).replace("/", ".").replace(".__init__", "").replace(".py", "")
+    name = re.sub(r"^api\.?", "", name)
+    logger = init_logging("api")
+    if name:
+        logger = logger.getChild(name)
+    return logger
+
+
+def get_traceback_info(tb):
+    """Prase the traceback information for a given traceback"""
+    tb_list = []
+    tb_id = 0
+    last_ui = None
+    while tb is not None:
+        f = tb.tb_frame
+        line_no = tb.tb_lineno
+        tb_list.append((f, line_no))
+        tb = tb.tb_next
+        if "/ui/" in f.f_code.co_filename:
+            last_ui = tb_id
+        tb_id += 1
+
+    if last_ui is not None:
+        tb_frame, line = tb_list[last_ui]
+        user = tb_frame.f_locals.get("kwargs", {}).get("user", None)
+
+        if not user:
+            temp = tb_frame.f_locals.get("_", {})
+            if isinstance(temp, dict):
+                user = temp.get("user", None)
+
+        if not user:
+            user = tb_frame.f_locals.get("user", None)
+
+        if not user:
+            user = tb_frame.f_locals.get("impersonator", None)
+
+        if user:
+            return user, tb_frame.f_code.co_filename, tb_frame.f_code.co_name, line
+
+        return None
+
+    return None
+
+
+def __dumb_log(log, msg, is_exception=False):
+    """Dumb logger for use with log_with_traceback"""
+    args: Union[str, bytes] = request.query_string
+    if isinstance(args, bytes):
+        args = args.decode()
+
+    if args:
+        args = f"?{args}"
+
+    message = f"{msg} - {request.path}{args}"
+    if is_exception:
+        log.exception(message)
+    else:
+        log.warning(message)
+
+
+def log_with_traceback(traceback, msg, is_exception=False, audit=False):
+    """Log a message along with the stacktrace"""
+    log = get_logger("traceback") if not audit else logging.getLogger("howler.api.audit")
+
+    tb_info = get_traceback_info(traceback)
+    if tb_info:
+        tb_user, tb_file, tb_function, tb_line_no = tb_info
+        args: Optional[Union[str, bytes]] = request.query_string
+        if args:
+            args = f"?{args if isinstance(args, str) else args.decode()}"
+        else:
+            args = ""
+
+        try:
+            message = (
+                f'{tb_user["uname"]} [{tb_user["classification"]}] :: {msg} - {tb_file}:{tb_function}:{tb_line_no}'
+                f'[{os.environ.get("HOWLER_VERSION", "0.0.0.dev0")}] ({request.path}{args})'
+            )
+            if is_exception:
+                log.exception(message)
+            else:
+                log.warning(message)
+        except Exception:
+            __dumb_log(log, msg, is_exception=is_exception)
+    else:
+        __dumb_log(log, msg, is_exception=is_exception)