clue-api 1.0.0.dev7__py3-none-any.whl

clue/common/logging/__init__.py

@@ -0,0 +1,291 @@
+import json
+import logging
+import logging.handlers
+import os
+import re
+import uuid
+from traceback import format_exception
+from typing import Optional, Self, Union
+
+from flask import request
+
+from clue.common.logging.format import (
+    BRL_DATE_FORMAT,
+    BRL_JSON_FORMAT,
+    BRL_LOG_FORMAT,
+    BRL_SYSLOG_FORMAT,
+)
+from clue.common.str_utils import default_string_value
+
+LOG_LEVEL_MAP = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+    "DISABLED": 60,
+}
+
+DEBUG = False
+
+
+class JsonFormatter(logging.Formatter):
+    """A custom implementation of logging.Formatter that supports json logs as well as traceback for exceptions.
+
+    Args:
+        logging (str): The template to use for the logs
+    """
+
+    def formatMessage(self: Self, record: logging.LogRecord):  # noqa: N802
+        """Formats record.message as json, and formats exceptions using traceback.
+
+        Args:
+            record (logging.LogRecord): The log record to format
+
+        Returns:
+            str: The formatted log
+        """
+        if record.exc_info:
+            record.exc_text = self.formatException(record.exc_info)
+            record.exc_info = None
+
+        if record.exc_text:
+            record.message += "\n" + record.exc_text
+            record.exc_text = None
+
+        record.message = json.dumps(record.message)
+        return self._style.format(record)
+
+    def formatException(self, exc_info):  # noqa: N802
+        """Formats the exception using traceback
+
+        Args:
+            exc_info (logging._SysExcInfoType): The exc_info object from the LogRecord
+
+        Returns:
+            str: The formatted traceback error
+        """
+        return "".join(format_exception(*exc_info))
+
+
+def init_logging(name: str, log_level: Optional[int] = None):  # noqa: C901
+    """Initializes the logging stack for an app
+
+    Args:
+        name (str): The name of the app
+        log_level (Optional[int], optional): The log level to use. Defaults to None.
+
+    Returns:
+        Logger: The initialized logger
+    """
+    from clue.config import config
+
+    app_name: str = default_string_value(env_name="APP_NAME", default="clue")  # type: ignore[assignment]
+
+    logger = logging.getLogger(app_name)
+
+    # Test if we've initialized the log handler already.
+    if len(logger.handlers) != 0:
+        return logger.getChild(name)
+
+    if name.startswith(f"{app_name}."):
+        name = name[len(app_name) + 1 :]
+
+    debug = config.api.debug
+    config.logging.log_to_console = config.logging.log_to_console or debug
+
+    if log_level is None:
+        log_level = LOG_LEVEL_MAP[config.logging.log_level]
+
+    logging.root.setLevel(logging.CRITICAL)
+    logger.setLevel(log_level)
+
+    if config.logging.log_level == "DISABLED":
+        # While log_level is set to disable, we will not create any handlers
+        return logger.getChild(name)
+
+    if config.logging.log_to_file:
+        if not os.path.isdir(config.logging.log_directory):
+            logger.warning(
+                "Log directory does not exist. Will try to create %s",
+                config.logging.log_directory,
+            )
+            os.makedirs(config.logging.log_directory)
+
+        if log_level <= logging.DEBUG:
+            dbg_file_handler = logging.handlers.RotatingFileHandler(
+                os.path.join(config.logging.log_directory, f"{name}.dbg"),
+                maxBytes=10485760,
+                backupCount=5,
+            )
+            dbg_file_handler.setLevel(logging.DEBUG)
+            if config.logging.log_as_json:
+                dbg_file_handler.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
+            else:
+                dbg_file_handler.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
+            logger.addHandler(dbg_file_handler)
+
+        if log_level <= logging.INFO:
+            op_file_handler = logging.handlers.RotatingFileHandler(
+                os.path.join(config.logging.log_directory, f"{name}.log"),
+                maxBytes=10485760,
+                backupCount=5,
+            )
+            op_file_handler.setLevel(logging.INFO)
+            if config.logging.log_as_json:
+                op_file_handler.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
+            else:
+                op_file_handler.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
+            logger.addHandler(op_file_handler)
+
+        if log_level <= logging.ERROR:
+            err_file_handler = logging.handlers.RotatingFileHandler(
+                os.path.join(config.logging.log_directory, f"{name}.err"),
+                maxBytes=10485760,
+                backupCount=5,
+            )
+            err_file_handler.setLevel(logging.ERROR)
+            if config.logging.log_as_json:
+                err_file_handler.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
+            else:
+                err_file_handler.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
+            logger.addHandler(err_file_handler)
+
+    if config.logging.log_to_console:
+        console = logging.StreamHandler()
+        if config.logging.log_as_json:
+            console.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
+        else:
+            console.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
+        logger.addHandler(console)
+
+    if config.logging.log_to_syslog and config.logging.syslog_host and config.logging.syslog_port:
+        syslog_handler = logging.handlers.SysLogHandler(
+            address=(config.logging.syslog_host, config.logging.syslog_port)
+        )
+        syslog_handler.formatter = logging.Formatter(BRL_SYSLOG_FORMAT)
+        logger.addHandler(syslog_handler)
+
+    return logger.getChild(name)
+
+
+def get_logger(name: Optional[str] = None, parent: Optional[logging.Logger] = None) -> logging.Logger:
+    """Gets the logger for a specified python file
+
+    Args:
+        name (Optional[str], optional): The name of the python file. Defaults to None.
+        parent (Optional[logging.Logger], optional): The parent of the file. Defaults to None.
+
+    Returns:
+        logging.Logger: _description_
+    """
+    if name:
+        name = (
+            re.sub(r".+clue(-api)?/", "", re.sub(r".+plugins/", "", name))
+            .replace("/", ".")
+            .replace(".__init__", "")
+            .replace(".py", "")
+        )
+        name = re.sub(r"^api\.?", "", name)
+
+    logger = parent or init_logging("api")
+
+    if name:
+        logger = logger.getChild(name)
+
+    return logger
+
+
+def get_traceback_info(tb):
+    """Gets the traceback info of a traceback"""
+    tb_list = []
+    tb_id = 0
+    last_ui = None
+    while tb is not None:
+        f = tb.tb_frame
+        line_no = tb.tb_lineno
+        tb_list.append((f, line_no))
+        tb = tb.tb_next
+        if "/ui/" in f.f_code.co_filename:
+            last_ui = tb_id
+        tb_id += 1
+
+    if last_ui is not None:
+        tb_frame, line = tb_list[last_ui]
+        user = tb_frame.f_locals.get("kwargs", {}).get("user", None)
+
+        if not user:
+            temp = tb_frame.f_locals.get("_", {})
+            if isinstance(temp, dict):
+                user = temp.get("user", None)
+
+        if not user:
+            user = tb_frame.f_locals.get("user", None)
+
+        if not user:
+            user = tb_frame.f_locals.get("impersonator", None)
+
+        if user:
+            return user, tb_frame.f_code.co_filename, tb_frame.f_code.co_name, line
+
+        return None
+
+    return None
+
+
+def dumb_log(log, msg, is_exception=False):
+    """Logs a message to a generic logger."""
+    args: Union[str, bytes] = request.query_string
+    if isinstance(args, bytes):
+        args = args.decode()
+
+    if args:
+        args = f"?{args}"
+
+    message = f"{msg} - {request.path}{args}"
+    if is_exception:
+        log.exception(message)
+    else:
+        log.warning(message)
+
+
+def log_with_traceback(traceback, msg, is_exception=False, audit=False):
+    """Logs a message with a traceback"""
+    log = get_logger("traceback") if not audit else logging.getLogger("clue.api.audit")
+
+    tb_info = get_traceback_info(traceback)
+    if tb_info:
+        tb_user, tb_file, tb_function, tb_line_no = tb_info
+        args: Optional[Union[str, bytes]] = request.query_string
+        if args:
+            args = f"?{args if isinstance(args, str) else args.decode()}"
+        else:
+            args = ""
+
+        # noinspection PyBroadException
+        try:
+            message = (
+                f'{tb_user["uname"]} [{tb_user["classification"]}] :: {msg} - {tb_file}:{tb_function}:{tb_line_no}'
+                f'[{os.environ.get("CLUE_VERSION", "0.0.0.dev0")}] ({request.path}{args})'
+            )
+            if is_exception:
+                log.exception(message)
+            else:
+                log.warning(message)
+        except Exception:
+            dumb_log(log, msg, is_exception=is_exception)
+    else:
+        dumb_log(log, msg, is_exception=is_exception)
+
+
+def log_error(logger, msg, err=None, status_code=None):
+    """Log a standard error string, with a unique id reference for logging."""
+    err_id = str(uuid.uuid4())
+    error = [msg]
+    if status_code:
+        error.append(f"{status_code=}")
+    if err:
+        error.append(f"{err=}")
+    error.append(f"{err_id}")
+    logger.error(" :: ".join(error))
+    return err_id

clue/common/logging/audit.py

@@ -0,0 +1,157 @@
+import logging
+import os
+import sys
+
+from flask import request
+
+from clue.common.logging.format import (
+    BRL_AUDIT_FORMAT,
+    BRL_DATE_FORMAT,
+    BRL_ISO_DATE_FORMAT,
+    BRL_LOG_FORMAT,
+)
+from clue.config import DEBUG, config
+
+AUDIT = config.api.audit
+
+AUDIT_KW_TARGET = [
+    "sid",
+    "sha256",
+    "copy_sid",
+    "filter",
+    "query",
+    "username",
+    "group",
+    "rev",
+    "wq_id",
+    "index",
+    "cache_key",
+    "alert_key",
+    "alert_id",
+    "url",
+    "q",
+    "fq",
+    "file_hash",
+    "heuristic_id",
+    "error_key",
+    "mac",
+    "vm_type",
+    "vm_name",
+    "config_name",
+    "servicename",
+    "vm",
+    "transition",
+    "data",
+    "id",
+    "comment_id",
+    "label_set",
+    "tool_name",
+    "operation_id",
+    "category",
+    "label",
+    "type_name",
+    "type",
+    "value",
+]
+
+AUDIT_LOG = logging.getLogger("clue.api.audit")
+AUDIT_LOG.propagate = False
+
+if AUDIT:
+    AUDIT_LOG.setLevel(logging.DEBUG)
+
+if not os.path.exists(config.logging.log_directory):
+    os.makedirs(config.logging.log_directory)
+
+fh = logging.FileHandler(os.path.join(config.logging.log_directory, "brl_audit.log"))
+fh.setLevel(logging.DEBUG)
+fh.setFormatter(
+    logging.Formatter(
+        BRL_LOG_FORMAT if DEBUG else BRL_AUDIT_FORMAT,
+        BRL_DATE_FORMAT if DEBUG else BRL_ISO_DATE_FORMAT,
+    )
+)
+AUDIT_LOG.addHandler(fh)
+
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.INFO)
+ch.setFormatter(
+    logging.Formatter(
+        BRL_LOG_FORMAT if DEBUG else BRL_AUDIT_FORMAT,
+        BRL_DATE_FORMAT if DEBUG else BRL_ISO_DATE_FORMAT,
+    )
+)
+AUDIT_LOG.addHandler(ch)
+
+#########################
+# End of prepare logger #
+#########################
+
+
+def audit(args, kwargs, user, func, impersonator=None):
+    """Audit
+
+    Arguments:
+
+    """
+    try:
+        json_blob = request.json
+        if not isinstance(json_blob, dict):
+            json_blob = {}
+    except Exception:
+        json_blob = {}
+
+    try:
+        req_args = ["%s='%s'" % (k, v) for k, v in request.args.items() if k in AUDIT_KW_TARGET]
+        method = request.method
+        path = request.path
+    except RuntimeError:
+        req_args = []
+        method = "N/A"
+        path = "N/A"
+
+    params_list = (
+        list(args)
+        + ["%s='%s'" % (k, v) for k, v in kwargs.items() if k in AUDIT_KW_TARGET]
+        + req_args
+        + ["%s='%s'" % (k, v) for k, v in json_blob.items() if k in AUDIT_KW_TARGET]
+    )
+
+    if impersonator:
+        audit_user = f"{impersonator} on behalf of {user['uname']}"
+    else:
+        audit_user = user["uname"]
+
+    if DEBUG:
+        # In debug mode, you'll get an output like:
+        # 23/03/20 14:26:56 DEBUG clue.api.audit | goose - search(index='...', query='...')
+        AUDIT_LOG.debug(
+            "%s [%s]- %s(%s)",
+            audit_user,
+            user["classification"],
+            func.__name__,
+            ", ".join(params_list),
+        )
+    else:
+        # In prod, you'll get an output like:
+        # {
+        #     "date": "2023-03-20T18:33:27-0400",
+        #     "type": "audit",
+        #     "app_name": "clue",
+        #     "api": "clue.api.audit",
+        #     "severity": "INFO",
+        #     "user": "goose",
+        #     "function": "search(index='hit', query='blah blah')",
+        #     "method": "POST",
+        #     "path": "/api/v1/search/hit/"
+        # }
+        AUDIT_LOG.info(
+            "",
+            extra={
+                "user": audit_user,
+                "function": f"{func.__name__}({', '.join(params_list)})",
+                "method": method,
+                "path": path,
+                "classification": user["classification"],
+            },
+        )

clue/common/logging/format.py

@@ -0,0 +1,42 @@
+import json
+
+from clue.common.str_utils import default_string_value
+
+hostname = "unknownhost"
+# noinspection PyBroadException
+try:
+    import socket
+
+    hostname = socket.gethostname()
+except Exception:  # noqa: S110
+    pass
+
+APP_NAME: str = default_string_value(env_name="APP_NAME", default="clue")  # type: ignore[assignment]
+
+BRL_SYSLOG_FORMAT = f"HWL %(levelname)8s {hostname} %(process)5d %(name)40s | %(message)s"
+BRL_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s | %(message)s"
+BRL_DATE_FORMAT = "%y/%m/%d %H:%M:%S"
+BRL_JSON_FORMAT = (
+    f"{{"
+    f'"@timestamp": "%(asctime)s", '
+    f'"event": {{ "module": "{APP_NAME}", "dataset": "%(name)s" }}, '
+    f'"host": {{ "hostname": "{hostname}" }}, '
+    f'"log": {{ "level": "%(levelname)s", "logger": "%(name)s" }}, '
+    f'"process": {{ "pid": "%(process)d" }}, '
+    f'"message": %(message)s}}'
+)
+BRL_ISO_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+BRL_AUDIT_FORMAT = json.dumps(
+    {
+        "date": "%(asctime)s",
+        "type": "audit",
+        "app_name": APP_NAME,
+        "api": f"{APP_NAME}.api.audit",
+        "severity": "%(levelname)s",
+        "user": "%(user)s",
+        "classification": "%(classification)s",
+        "function": "%(function)s",
+        "method": "%(method)s",
+        "path": "%(path)s",
+    }
+).replace('"msg"', "%(message)s")

clue/common/regex.py

@@ -0,0 +1,31 @@
+DOMAIN_REGEX = r"(?:(?=[a-z0-9-]{1,63}\.)(?:xn--)?[a-z0-9-]+(?:-[a-z0-9]+)*\.)+[a-z]{2,63}"
+DOMAIN_ONLY_REGEX = f"^{DOMAIN_REGEX}$"
+DOMAIN_EXCLUDED_NORM_CHARS = "./?@#"
+EMAIL_REGEX = f"^[a-zA-Z0-9!#$%&'*+/=?^_‘{{|}}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_‘{{|}}~-]+)*@({DOMAIN_REGEX})$"
+IPV4_REGEX = r"(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
+IPV6_REGEX = (
+    r"(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|"
+    r"(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|"
+    r"(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|"
+    r"(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|"
+    r":(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|"
+    r"::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|"
+    r"(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|"
+    r"(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))"
+)
+IP_REGEX = f"(?:{IPV4_REGEX}|{IPV6_REGEX})"
+IP_ONLY_REGEX = f"^{IP_REGEX}$"
+IPV4_ONLY_REGEX = f"^{IPV4_REGEX}$"
+IPV6_ONLY_REGEX = f"^{IPV6_REGEX}$"
+PORT_REGEX = r"^(0|[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$"
+MD5_REGEX = r"^[a-fA-F0-9]{32}$"
+SHA1_REGEX = r"^[a-fA-F0-9]{40}$"
+SHA256_REGEX = r"^[a-fA-F0-9]{64}$"
+URI_PATH = r"([/?#]\S*)"
+URI_REGEX = (
+    f"((?:(?:[A-Za-z][A-Za-z0-9+.-]*:)//)(?:[^/?#\\s]+@)?({IP_REGEX}|{DOMAIN_REGEX})(?::\\d{{1,5}})?{URI_PATH}?)"
+)
+URI_ONLY = f"^{URI_REGEX}$"
+
+EMAIL_PATH_REGEX = r"^[A-Z]*_EMAIL://.*"
+HBS_AGENT_ID_REGEX = r"[0-9a-fA-F]{1,4}.[0-9a-fA-F]{1,4}.[0-9a-fA-F]{1,4}.[0-9a-fA-F]{1,4}"