howler-api 2.13.0.dev329__py3-none-any.whl

howler/common/logging/__init__.py

@@ -0,0 +1,241 @@
+import json
+import logging
+import logging.handlers
+import os
+import re
+from traceback import format_exception
+from typing import TYPE_CHECKING, Optional, Union
+
+from flask import request
+from typing_extensions import override  # type: ignore
+
+from howler.common import loader
+from howler.common.logging.format import (
+    HWL_DATE_FORMAT,
+    HWL_JSON_FORMAT,
+    HWL_LOG_FORMAT,
+    HWL_SYSLOG_FORMAT,
+)
+
+if TYPE_CHECKING:
+    from howler.odm.models.config import Config
+
+LOG_LEVEL_MAP = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+    "DISABLED": 60,
+}
+
+DEBUG = False
+
+
+class JsonFormatter(logging.Formatter):
+    """logging Formatter to output in JSON"""
+
+    @override
+    def formatMessage(self, record):
+        if record.exc_info:
+            record.exc_text = self.formatException(record.exc_info)
+            record.exc_info = None
+
+        if record.exc_text:
+            record.message += "\n" + record.exc_text
+            record.exc_text = None
+
+        record.message = json.dumps(record.message)
+        return self._style.format(record)
+
+    @override
+    def formatException(self, exc_info):
+        return "".join(format_exception(*exc_info))
+
+
+def init_log_to_file(logger: logging.Logger, log_level: int, name: str, config: "Config"):
+    """Initialize file-based logging"""
+    if not os.path.isdir(config.logging.log_directory):
+        logger.warning(
+            "Log directory does not exist. Will try to create %s",
+            config.logging.log_directory,
+        )
+        os.makedirs(config.logging.log_directory)
+
+    if log_level <= logging.DEBUG:
+        dbg_file_handler = logging.handlers.RotatingFileHandler(
+            os.path.join(config.logging.log_directory, f"{name}.dbg"),
+            maxBytes=10485760,
+            backupCount=5,
+        )
+        dbg_file_handler.setLevel(logging.DEBUG)
+        if config.logging.log_as_json:
+            dbg_file_handler.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            dbg_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(dbg_file_handler)
+
+    if log_level <= logging.INFO:
+        op_file_handler = logging.handlers.RotatingFileHandler(
+            os.path.join(config.logging.log_directory, f"{name}.log"),
+            maxBytes=10485760,
+            backupCount=5,
+        )
+        op_file_handler.setLevel(logging.INFO)
+        if config.logging.log_as_json:
+            op_file_handler.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            op_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(op_file_handler)
+
+    if log_level <= logging.ERROR:
+        err_file_handler = logging.handlers.RotatingFileHandler(
+            os.path.join(config.logging.log_directory, f"{name}.err"),
+            maxBytes=10485760,
+            backupCount=5,
+        )
+        err_file_handler.setLevel(logging.ERROR)
+        if config.logging.log_as_json:
+            err_file_handler.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            err_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        err_file_handler.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(err_file_handler)
+
+
+def init_logging(name: str, log_level: Optional[int] = None):
+    """Initialize the logger"""
+    from howler.config import config
+
+    logger = logging.getLogger(loader.APP_NAME)
+
+    # Test if we've initialized the log handler already.
+    if len(logger.handlers) != 0:
+        return logger.getChild(name)
+
+    if name.startswith(f"{loader.APP_NAME}."):
+        name = name[len(loader.APP_NAME) + 1 :]
+
+    config.logging.log_to_console = config.logging.log_to_console or config.ui.debug
+
+    if log_level is None:
+        log_level = LOG_LEVEL_MAP[config.logging.log_level]
+
+    logging.root.setLevel(logging.CRITICAL)
+    logger.setLevel(log_level)
+
+    if config.logging.log_level == "DISABLED":
+        # While log_level is set to disable, we will not create any handlers
+        return logger.getChild(name)
+
+    if config.logging.log_to_file:
+        init_log_to_file(logger, log_level, name, config)
+
+    if config.logging.log_to_console:
+        console = logging.StreamHandler()
+        if config.logging.log_as_json:
+            console.setFormatter(JsonFormatter(HWL_JSON_FORMAT))
+        else:
+            console.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+        logger.addHandler(console)
+
+    if config.logging.log_to_syslog and config.logging.syslog_host and config.logging.syslog_port:
+        syslog_handler = logging.handlers.SysLogHandler(
+            address=(config.logging.syslog_host, config.logging.syslog_port)
+        )
+        syslog_handler.formatter = logging.Formatter(HWL_SYSLOG_FORMAT)
+        logger.addHandler(syslog_handler)
+
+    logger.debug("Logger ready!")
+    return logger.getChild(name)
+
+
+def get_logger(name: str = "default") -> logging.Logger:
+    """Get a logger with a useful name given a filename"""
+    name = re.sub(r".+(howler|test)/", "", name).replace("/", ".").replace(".__init__", "").replace(".py", "")
+    name = re.sub(r"^api\.?", "", name)
+    logger = init_logging("api")
+    if name:
+        logger = logger.getChild(name)
+    return logger
+
+
+def get_traceback_info(tb):
+    """Prase the traceback information for a given traceback"""
+    tb_list = []
+    tb_id = 0
+    last_ui = None
+    while tb is not None:
+        f = tb.tb_frame
+        line_no = tb.tb_lineno
+        tb_list.append((f, line_no))
+        tb = tb.tb_next
+        if "/ui/" in f.f_code.co_filename:
+            last_ui = tb_id
+        tb_id += 1
+
+    if last_ui is not None:
+        tb_frame, line = tb_list[last_ui]
+        user = tb_frame.f_locals.get("kwargs", {}).get("user", None)
+
+        if not user:
+            temp = tb_frame.f_locals.get("_", {})
+            if isinstance(temp, dict):
+                user = temp.get("user", None)
+
+        if not user:
+            user = tb_frame.f_locals.get("user", None)
+
+        if not user:
+            user = tb_frame.f_locals.get("impersonator", None)
+
+        if user:
+            return user, tb_frame.f_code.co_filename, tb_frame.f_code.co_name, line
+
+        return None
+
+    return None
+
+
+def __dumb_log(log, msg, is_exception=False):
+    """Dumb logger for use with log_with_traceback"""
+    args: Union[str, bytes] = request.query_string
+    if isinstance(args, bytes):
+        args = args.decode()
+
+    if args:
+        args = f"?{args}"
+
+    message = f"{msg} - {request.path}{args}"
+    if is_exception:
+        log.exception(message)
+    else:
+        log.warning(message)
+
+
+def log_with_traceback(traceback, msg, is_exception=False, audit=False):
+    """Log a message along with the stacktrace"""
+    log = get_logger("traceback") if not audit else logging.getLogger("howler.api.audit")
+
+    tb_info = get_traceback_info(traceback)
+    if tb_info:
+        tb_user, tb_file, tb_function, tb_line_no = tb_info
+        args: Optional[Union[str, bytes]] = request.query_string
+        if args:
+            args = f"?{args if isinstance(args, str) else args.decode()}"
+        else:
+            args = ""
+
+        try:
+            message = (
+                f'{tb_user["uname"]} [{tb_user["classification"]}] :: {msg} - {tb_file}:{tb_function}:{tb_line_no}'
+                f'[{os.environ.get("HOWLER_VERSION", "0.0.0.dev0")}] ({request.path}{args})'
+            )
+            if is_exception:
+                log.exception(message)
+            else:
+                log.warning(message)
+        except Exception:
+            __dumb_log(log, msg, is_exception=is_exception)
+    else:
+        __dumb_log(log, msg, is_exception=is_exception)

howler/common/logging/audit.py

@@ -0,0 +1,138 @@
+import logging
+import os
+import sys
+
+from flask import request
+
+from howler.common.logging.format import HWL_AUDIT_FORMAT, HWL_DATE_FORMAT, HWL_ISO_DATE_FORMAT, HWL_LOG_FORMAT
+from howler.config import DEBUG, config
+
+AUDIT = config.ui.audit
+
+AUDIT_KW_TARGET = [
+    "sid",
+    "sha256",
+    "copy_sid",
+    "filter",
+    "query",
+    "username",
+    "group",
+    "rev",
+    "wq_id",
+    "index",
+    "cache_key",
+    "alert_key",
+    "alert_id",
+    "url",
+    "q",
+    "fq",
+    "file_hash",
+    "heuristic_id",
+    "error_key",
+    "mac",
+    "vm_type",
+    "vm_name",
+    "config_name",
+    "servicename",
+    "vm",
+    "transition",
+    "data",
+    "id",
+    "comment_id",
+    "label_set",
+    "tool_name",
+    "operation_id",
+    "category",
+    "label",
+]
+
+AUDIT_LOG = logging.getLogger("howler.api.audit")
+AUDIT_LOG.propagate = False
+
+if AUDIT:
+    AUDIT_LOG.setLevel(logging.DEBUG)
+
+if not os.path.exists(config.logging.log_directory):
+    os.makedirs(config.logging.log_directory)
+
+fh = logging.FileHandler(os.path.join(config.logging.log_directory, "hwl_audit.log"))
+fh.setLevel(logging.DEBUG)
+fh.setFormatter(
+    logging.Formatter(
+        HWL_LOG_FORMAT if DEBUG else HWL_AUDIT_FORMAT,
+        HWL_DATE_FORMAT if DEBUG else HWL_ISO_DATE_FORMAT,
+    )
+)
+AUDIT_LOG.addHandler(fh)
+
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.INFO)
+ch.setFormatter(
+    logging.Formatter(
+        HWL_LOG_FORMAT if DEBUG else HWL_AUDIT_FORMAT,
+        HWL_DATE_FORMAT if DEBUG else HWL_ISO_DATE_FORMAT,
+    )
+)
+AUDIT_LOG.addHandler(ch)
+
+#########################
+# End of prepare logger #
+#########################
+
+
+def audit(args, kwargs, logged_in_uname, user, func, impersonator=None):
+    """Log audit information for a given function executed by a given user."""
+    try:
+        json_blob = request.json
+        if not isinstance(json_blob, dict):
+            json_blob = {}
+    except Exception:
+        json_blob = {}
+
+    try:
+        req_args = ["%s='%s'" % (k, v) for k, v in request.args.items() if k in AUDIT_KW_TARGET]
+    except RuntimeError:
+        req_args = []
+
+    params_list = (
+        list(args)
+        + ["%s='%s'" % (k, v) for k, v in kwargs.items() if k in AUDIT_KW_TARGET]
+        + req_args
+        + ["%s='%s'" % (k, v) for k, v in json_blob.items() if k in AUDIT_KW_TARGET]
+    )
+
+    if impersonator:
+        audit_user = f"{impersonator} on behalf of {logged_in_uname}"
+    else:
+        audit_user = logged_in_uname
+    if DEBUG:
+        # In debug mode, you'll get an output like:
+        # 23/03/20 14:26:56 DEBUG howler.api.audit | goose - search(index='...', query='...')
+        AUDIT_LOG.debug(
+            "%s - %s(%s)",
+            audit_user,
+            func.__name__,
+            ", ".join(params_list),
+        )
+    else:
+        # In prod, you'll get an output like:
+        # {
+        #     "date": "2023-03-20T18:33:27-0400",
+        #     "type": "audit",
+        #     "app_name": "howler",
+        #     "api": "howler.api.audit",
+        #     "severity": "INFO",
+        #     "user": "goose",
+        #     "function": "search(index='hit', query='howler.escalation:alert AND howler.status:open')",
+        #     "method": "POST",
+        #     "path": "/api/v1/search/hit/"
+        # }
+        AUDIT_LOG.info(
+            "",
+            extra={
+                "user": audit_user,
+                "function": f"{func.__name__}({', '.join(params_list)})",
+                "method": request.method,
+                "path": request.path,
+            },
+        )

howler/common/logging/format.py

@@ -0,0 +1,38 @@
+import json
+import os
+
+APP_NAME = os.environ.get("APP_NAME", "howler")
+
+try:
+    from howler.common.net import get_hostname
+
+    hostname = get_hostname()
+except Exception:
+    hostname = "unknownhost"
+
+HWL_SYSLOG_FORMAT = f"HWL %(levelname)8s {hostname} %(process)5d %(name)40s | %(message)s"
+HWL_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s | %(message)s"
+HWL_DATE_FORMAT = "%y/%m/%d %H:%M:%S"
+HWL_JSON_FORMAT = (
+    f"{{"
+    f'"@timestamp": "%(asctime)s", '
+    f'"event": {{ "module": "{APP_NAME}", "dataset": "%(name)s" }}, '
+    f'"host": {{ "hostname": "{hostname}" }}, '
+    f'"log": {{ "level": "%(levelname)s", "logger": "%(name)s" }}, '
+    f'"process": {{ "pid": "%(process)d" }}, '
+    f'"message": %(message)s}}'
+)
+HWL_ISO_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+HWL_AUDIT_FORMAT = json.dumps(
+    {
+        "date": "%(asctime)s",
+        "type": "audit",
+        "app_name": APP_NAME,
+        "api": "howler.api.audit",
+        "severity": "%(levelname)s",
+        "user": "%(user)s",
+        "function": "%(function)s",
+        "method": "%(method)s",
+        "path": "%(path)s",
+    }
+).replace('"msg"', "%(message)s")

howler/common/net.py

@@ -0,0 +1,79 @@
+import socket
+import uuid
+from ipaddress import IPv4Network, ip_address
+from typing import Union
+
+from howler.common.net_static import TLDS_ALPHA_BY_DOMAIN
+
+
+def is_valid_port(value: Union[int, str, float]) -> bool:
+    "Check if a port is valid"
+    try:
+        if 1 <= int(value) <= 65535:
+            return True
+    except ValueError:
+        pass
+
+    return False
+
+
+def is_valid_domain(domain: str) -> bool:
+    "Check if a domain is valid"
+    if "@" in domain:
+        return False
+
+    if "." in domain:
+        tld = domain.split(".")[-1]
+        return tld.upper() in TLDS_ALPHA_BY_DOMAIN
+
+    return False
+
+
+def is_valid_ip(ip: str) -> bool:
+    "Check if an ip is valid"
+    parts = ip.split(".")
+    if len(parts) == 4:
+        for p in parts:
+            try:
+                if not (0 <= int(p) <= 255):
+                    return False
+            except ValueError:
+                return False
+
+        if int(parts[0]) == 0:
+            return False
+
+        if int(parts[3]) == 0:
+            return False
+
+        return True
+
+    return False
+
+
+def is_ip_in_network(ip: str, network: IPv4Network) -> bool:
+    "Check if an ip is in a given network"
+    if not is_valid_ip(ip):
+        return False
+
+    return ip_address(ip) in network
+
+
+def is_valid_email(email: str) -> bool:
+    "Check if an email is valid"
+    parts = email.split("@")
+    if len(parts) == 2:
+        if is_valid_domain(parts[1]):
+            return True
+
+    return False
+
+
+def get_hostname() -> str:
+    "Get the hostname of the computer howler is running on"
+    return socket.gethostname()
+
+
+def get_mac_address() -> str:
+    "Get the mac address of the computer howler is running on"
+    return "".join(["{0:02x}".format((uuid.getnode() >> i) & 0xFF) for i in range(0, 8 * 6, 8)][::-1]).upper()