clue-api 1.0.0.dev7__py3-none-any.whl

clue/common/str_utils.py

@@ -0,0 +1,213 @@
+import os
+import re
+from copy import copy
+from typing import Optional, Self, Union
+
+from clue.common.exceptions import ClueAttributeError
+
+
+def remove_bidir_unicode_controls(in_str):
+    """Remove UBA characters"""
+    try:
+        no_controls_str = "".join(
+            c
+            for c in in_str
+            if c
+            not in [
+                "\u202e",
+                "\u202b",
+                "\u202d",
+                "\u202a",
+                "\u200e",
+                "\u200f",
+            ]
+        )
+    except Exception:
+        no_controls_str = in_str
+
+    return no_controls_str
+
+
+def wrap_bidir_unicode_string(uni_str):
+    """Wraps str in a LRE (Left-to-Right Embed) unicode control.
+
+    Guarantees that str can be concatenated to other strings without
+    affecting their left-to-right direction
+    """
+    if len(uni_str) == 0 or isinstance(uni_str, bytes):  # Not str, return it unchanged
+        return uni_str
+
+    re_obj = re.search(r"[\u202E\u202B\u202D\u202A\u200E\u200F]", uni_str)
+    if re_obj is None or len(re_obj.group()) == 0:  # No unicode bidir controls found, return string unchanged
+        return uni_str
+
+    # Parse str for unclosed bidir blocks
+    count = 0
+    for letter in uni_str:
+        if letter in ["\u202a", "\u202b", "\u202d", "\u202e"]:  # bidir block open?
+            count += 1
+        elif letter == "\u202c":
+            if count > 0:
+                count -= 1
+
+    # close all bidir blocks
+    if count > 0:
+        uni_str += "\u202c" * count
+
+        # Final wrapper (LTR block) to neutralize any Marks (u+200E and u+200F)
+    uni_str = "\u202a" + uni_str + "\u202c"
+
+    return uni_str
+
+
+# According to wikipedia, RFC 3629 restricted UTF-8 to end at U+10FFFF.
+# This removed the 6, 5 and (irritatingly) half of the 4 byte sequences.
+#
+# The start byte for 2-byte sequences should be a value between 0xc0 and
+# 0xdf but the values 0xc0 and 0xc1 are invalid as they could only be
+# the result of an overlong encoding of basic ASCII characters. There
+# are similar restrictions on the valid values for 3 and 4-byte sequences.
+_valid_utf8 = re.compile(
+    rb"""((?:
+    [\x09\x0a\x20-\x7e]|             # 1-byte (ASCII excluding control chars).
+    [\xc2-\xdf][\x80-\xbf]|          # 2-bytes (excluding overlong sequences).
+    [\xe0][\xa0-\xbf][\x80-\xbf]|    # 3-bytes (excluding overlong sequences).
+
+    [\xe1-\xec][\x80-\xbf]{2}|       # 3-bytes.
+    [\xed][\x80-\x9f][\x80-\xbf]|    # 3-bytes (up to invalid code points).
+    [\xee-\xef][\x80-\xbf]{2}|       # 3-bytes (after invalid code points).
+
+    [\xf0][\x90-\xbf][\x80-\xbf]{2}| # 4-bytes (excluding overlong sequences).
+    [\xf1-\xf3][\x80-\xbf]{3}|       # 4-bytes.
+    [\xf4][\x80-\x8f][\x80-\xbf]{2}  # 4-bytes (up to U+10FFFF).
+    )+)""",
+    re.VERBOSE,
+)
+
+
+def _escape(t, reversible=True):
+    if t[0] % 2:
+        return t[1].replace(b"\\", b"\\\\") if reversible else t[1]
+    else:
+        return b"".join((b"\\x%02x" % x) for x in t[1])
+
+
+def dotdump(s: Union[bytes, str]):
+    """Remove any non-ascii characters and replace them with periods
+
+    https://www.cs.cmu.edu/~pattis/15-1XX/common/handouts/ascii.html
+    """
+    if isinstance(s, str):
+        s = s.encode()
+
+    return "".join(["." if x < 32 or x > 126 else chr(x) for x in s])
+
+
+def escape_str(s, reversible=True, force_str=False):
+    """Escape a string"""
+    if isinstance(s, bytes):
+        return escape_str_strict(s, reversible)
+    elif not isinstance(s, str):
+        if force_str:
+            return str(s)
+        return s
+
+    try:
+        return escape_str_strict(
+            s.encode("utf-16", "surrogatepass").decode("utf-16").encode("utf-8"),
+            reversible,
+        )
+    except Exception:
+        return escape_str_strict(s.encode("utf-8", errors="backslashreplace"), reversible)
+
+
+# Returns a string (str) with only valid UTF-8 byte sequences.
+def escape_str_strict(s: bytes, reversible: bool = True) -> str:
+    """Strictly escape a string"""
+    escaped = b"".join([_escape(t, reversible) for t in enumerate(_valid_utf8.split(s))])
+    return escaped.decode("utf-8")
+
+
+def safe_str(s, force_str=False):
+    """Create a safe, escaped string"""
+    return escape_str(s, reversible=False, force_str=force_str)
+
+
+def is_safe_str(s: str) -> bool:
+    """Check if a given string is safe"""
+    return escape_str(copy(s), reversible=False) == s
+
+
+# This method not really necessary. More to stop people from rolling their own.
+def unescape_str(s):
+    """unescape a string"""
+    return s.decode("string_escape")
+
+
+def default_string_value(
+    *values: Optional[str], env_name: Optional[str] = None, default: Optional[str] = None
+) -> Optional[str]:
+    """Return a string value based on a list of potential values, an environmnet variable, or a default string"""
+    return next(
+        (val for val in values if val), (os.getenv(env_name, default or "") or default) if env_name else default
+    )
+
+
+def get_parent_key(key: str) -> str:
+    """Get a parent key of a key in the format a.b.c"""
+    return re.sub(r"^(.+)\..+?$", r"\1", key)
+
+
+def sanitize_lucene_query(query: str):
+    """Take in a given string, and escape it to ensure it is safe to search on via lucene"""
+    query = re.sub(r'([\^"~*?:\\/()[\]{}\-!])', r"\\\1", query)
+
+    return query.replace("&&", "\\&&").replace("||", "\\||")
+
+
+class NamedConstants(object):
+    """A class containing a list of named constants, as well as a reverse map for those constants to their name"""
+
+    def __init__(self: Self, name: str, string_value_list: list[tuple[str, str]]):
+        self._name = name
+        self._value_map = dict(string_value_list)
+        self._reverse_map = dict([(s[1], s[0]) for s in string_value_list])
+
+        # we also import the list as attributes so things like
+        # tab completion and introspection still work.
+        for s, v in self._value_map.items():
+            setattr(self, s, v)
+
+    def name_for_value(self, v):
+        """Get the name of a given value"""
+        return self._reverse_map[v]
+
+    def contains_value(self, v):
+        """Chgeck if this instance contains the given value"""
+        return v in self._reverse_map
+
+    def __getitem__(self, s):
+        return self._value_map[s]
+
+    def __getattr__(self, s):
+        # We implement our own getattr mainly to provide the better exception.
+        return self._value_map[s]
+
+
+class StringTable(NamedConstants):
+    """A subclass of NamedConstants that throws an attribute error if the value does not exist in the table"""
+
+    def contains_string(self, s):
+        """Chgeck if this instance contains the given value"""
+        return s in self._reverse_map
+
+    def __getitem__(self, s):
+        if s in self._value_map:
+            return s
+        raise ClueAttributeError("Invalid value for %s (%s)" % (self._name, s))
+
+    def __getattr__(self, s):
+        # We implement our own getattr mainly to provide the better exception.
+        if s in self._value_map:
+            return s
+        raise ClueAttributeError("Invalid value for %s (%s)" % (self._name, s))

clue/common/swagger.py

@@ -0,0 +1,139 @@
+import inspect
+import re
+from functools import wraps
+from typing import Any, Callable, Optional, cast
+
+from flasgger import utils
+from pydantic import TypeAdapter
+
+from clue.models.network import (
+    Annotation,
+    ClueResponse,
+    QueryEntry,
+    QueryResult,
+)
+from clue.services import type_service
+
+
+def monkey_patched_parse(obj: object, *args: tuple[Any], **kwargs) -> tuple[Optional[str], Optional[str], None]:
+    """Parse existing docstrings for a python object and return a short and long description of it
+
+    Args:
+        obj (object): The object to inspect.
+
+    Returns:
+        tuple[str, str, None]: A tuple containing the short and long description, along with a None just for fun
+            (i've got no idea why).
+    """
+    short_desc: Optional[str] = None
+    long_desc: Optional[str] = None
+
+    doc = inspect.getdoc(obj)
+
+    if doc:
+        short_desc = doc.splitlines()[0]
+        long_desc = f"```\n{doc}\n```"
+
+    return short_desc, long_desc, None
+
+
+utils.parse_docstring = monkey_patched_parse
+
+DEFINITIONS = {
+    "QueryResult": QueryResult.model_json_schema(ref_template="#/definitions/{model}"),
+    "QueryEntry": QueryEntry.model_json_schema(ref_template="#/definitions/{model}"),
+    "Annotation": Annotation.model_json_schema(ref_template="#/definitions/{model}"),
+}
+
+RESPONSES = {
+    status_code: {
+        "description": "Something went wrong with your request",
+        "schema": {
+            **ClueResponse.model_json_schema(),
+            "example": ClueResponse(error_message="Example error", status_code=status_code).model_dump(),
+        },
+    }
+    for status_code in [400, 401, 403, 404]
+}
+
+
+def generate_swagger_docs(responses: dict[int, str] = {}) -> Callable:  # noqa: C901
+    """Generates a decorator that allows to create swagger doc for an endpoint.
+
+    Args:
+        responses (dict[int, str], optional): A dict of the possible responses, with the HTTP code as the key and the
+            description of the response as the value. Defaults to {}.
+
+    Returns:
+        Callable: The decorator
+    """
+
+    def decorator(function: Callable) -> Callable:
+        func_signature = inspect.signature(function)
+        func_doc = inspect.getdoc(function)
+        if module := inspect.getmodule(function):
+            module_name = module.__name__
+        func_path = f"{module_name}.{function.__name__}" if module_name else function.__name__
+
+        path_params = [
+            {
+                "name": param,
+                "in": "path",
+                "type": "string",
+                "enum": list(type_service.SUPPORTED_TYPES.keys()) if param == "type_name" else None,
+            }
+            for param in func_signature.parameters
+            if param not in ["kwargs", "_"] and not param.startswith("_")
+        ]
+
+        query_params: list[dict[str, Any]] = []
+        if func_doc:
+            for section in func_doc.split("\n\n"):
+                lines = section.splitlines()
+                if not lines[0].lower().endswith("arguments:"):
+                    continue
+
+                lines = [re.sub(r" =>.+", "", line).strip() for line in lines[1:]]
+
+                for line in lines:
+                    if line.lower() == "none" or "=>" not in line:
+                        continue
+
+                    if ": " in line:
+                        name, type = line.split(": ")
+                    else:
+                        name = line
+                        type = None
+
+                    query_params.append({"name": name, "in": "query", "type": type})
+
+        tags: list[str] = []
+        if module := inspect.getmodule(function):
+            tags.append(module.__name__.split(".")[-1].capitalize())
+
+        cast(Any, function).specs_dict = {
+            "parameters": [*path_params, *query_params],
+            "definitions": DEFINITIONS,
+            "responses": {
+                "200": {
+                    "description": responses.get(200, "Request succeeded"),
+                    "schema": (
+                        TypeAdapter(func_signature.return_annotation).json_schema(ref_template="#/definitions/{model}")
+                        if func_signature.return_annotation != inspect._empty
+                        else None
+                    ),
+                },
+                **RESPONSES,
+            },
+            "summary": "test",
+            "tags": tags,
+            "operationId": func_path,
+        }
+
+        @wraps(function)
+        def wrapper(*args, **kwargs):
+            return function(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

clue/common/uid.py

@@ -0,0 +1,47 @@
+import hashlib
+import uuid
+from typing import Any, Literal, Optional
+
+TINY = 8
+SHORT = 16
+MEDIUM = NORMAL = 32
+LONG = 64
+
+
+def get_random_id() -> str:
+    """Generates a random unique id, using uuid4 and encoded in base62
+
+    Returns:
+        str: Base62 encoded uuid4
+    """
+    import baseconv
+
+    return baseconv.base62.encode(uuid.uuid4().int)
+
+
+def get_id_from_data(data: Any, prefix: Optional[str] = None, length: Literal[8, 16, 32, 64] = MEDIUM):  # type: ignore[assignment]
+    """Generates an id based on the provided data, using sha256, truncated to the specified length and encoded in base62
+
+    Args:
+        data (Any): The data to use to generate the id
+        prefix (Optional[str], optional): Defaults to None.
+        length (Literal[8, 16, 32, 64], optional): Defaults to 32.
+
+    Raises:
+        ValueError: Raised when an invalid length is provided
+
+    Returns:
+        str: The generated base62 encoded truncated sha256 hash.
+    """
+    import baseconv
+
+    possible_len = [TINY, SHORT, MEDIUM, LONG]
+    if length not in possible_len:
+        raise ValueError(f"Invalid hash length of {length}. Possible values are: {str(possible_len)}.")
+    sha256_hash = hashlib.sha256(str(data).encode()).hexdigest()[:length]
+    _hash = baseconv.base62.encode(int(sha256_hash, 16))
+
+    if isinstance(prefix, str):
+        _hash = f"{prefix}_{_hash}"
+
+    return _hash

clue/config.py

@@ -0,0 +1,60 @@
+import os
+
+from clue.common import forge
+from clue.models.config import Config
+
+config: Config = Config()
+
+#################################################################
+# Configuration
+
+CLASSIFICATION = forge.get_classification()
+
+AUDIT = config.api.audit
+
+SECRET_KEY = config.api.secret_key
+DEBUG = config.api.debug
+MAX_CLASSIFICATION = CLASSIFICATION.UNRESTRICTED
+
+USER_TYPES = {"admin", "user"}
+
+
+def get_version() -> str:
+    """The version of the Clue API
+
+    Returns:
+        str: The clue version
+    """
+    return os.environ.get("CLUE_VERSION", "1.0.0.dev0")
+
+
+def get_commit() -> str:
+    """The commit of the currently deployed Clue API
+
+    Returns:
+        str: The commit of the currently deployed image
+    """
+    return os.environ.get("COMMIT_HASH", "this is not the commit you are looking for")
+
+
+def get_branch() -> str:
+    """The branch of the current Clue Image
+
+    Returns:
+        str: The current branch
+    """
+    return os.environ.get("BRANCH", "this is not the branch you are looking for")
+
+
+def get_redis():
+    """The Redis instance used by Clue.
+
+    Returns:
+        The Redis client instance.
+    """
+    from clue.remote.datatypes import get_client
+
+    return get_client(config.core.redis.host, config.core.redis.port, False)
+
+
+cache = forge.cache

clue/constants/supported_types.py

@@ -0,0 +1,38 @@
+from clue.common.regex import (
+    DOMAIN_ONLY_REGEX,
+    EMAIL_PATH_REGEX,
+    EMAIL_REGEX,
+    HBS_AGENT_ID_REGEX,
+    IPV4_ONLY_REGEX,
+    IPV6_ONLY_REGEX,
+    MD5_REGEX,
+    PORT_REGEX,
+    SHA1_REGEX,
+    SHA256_REGEX,
+    URI_ONLY,
+)
+
+SUPPORTED_TYPES = {
+    "ipv4": IPV4_ONLY_REGEX,
+    "ipv6": IPV6_ONLY_REGEX,
+    # We don't auto-detect ip types, as it's redundant with ipv4/v6. This is just a convenience/backwards compat thing
+    "ip": None,
+    "domain": DOMAIN_ONLY_REGEX,
+    "port": PORT_REGEX,
+    "url": URI_ONLY,
+    "userid": None,
+    "user_agent": None,
+    "email_address": EMAIL_REGEX,
+    "email_id": None,
+    "email_path": EMAIL_PATH_REGEX,
+    "md5": MD5_REGEX,
+    "sha1": SHA1_REGEX,
+    "sha256": SHA256_REGEX,
+    "hbs_oid": None,
+    "hbs_agent_id": HBS_AGENT_ID_REGEX,
+    "telemetry": None,
+    "howler_id": None,
+    "hostname": None,
+}
+
+CASE_INSENSITIVE_TYPES = ["ip", "domain", "port"]

clue/cronjobs/__init__.py

@@ -0,0 +1,30 @@
+import importlib
+import os
+from pathlib import Path
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from pytz import timezone
+
+from clue.common.logging import get_logger
+
+logger = get_logger(__file__)
+
+scheduler = BackgroundScheduler(timezone=timezone(os.getenv("SCHEDULER_TZ", "America/Toronto")))
+
+
+def setup_jobs():
+    """Imports all modules in the current directory (cronjobs) and adds them to the scheduler."""
+    module_path = Path(__file__).parent
+    modules_to_import = [
+        _file for _file in module_path.iterdir() if _file.suffix == ".py" and _file.name != "__init__.py"
+    ]
+
+    for module in modules_to_import:
+        try:
+            job = importlib.import_module(f"clue.cronjobs.{module.stem}")
+
+            job.setup_job(scheduler)
+        except Exception as e:
+            logger.critical("Error when initializing %s - %s", module, e)
+
+    scheduler.start()

clue/cronjobs/plugins.py

@@ -0,0 +1,32 @@
+from apscheduler.schedulers.base import BaseScheduler
+from gevent.queue import Queue
+
+from clue.api.v1.registration import EXTERNAL_PLUGIN_SET
+from clue.common.logging import get_logger
+from clue.config import config
+from clue.models.config import ExternalSource
+
+logger = get_logger(__file__)
+
+config_updates = Queue()
+
+__scheduler_instance: BaseScheduler | None = None
+
+
+def update_external_source_list():
+    """Updates the external_sources list with the plugins that have been registered through the API."""
+    plugin_list: list[ExternalSource] = [ExternalSource.model_validate(item) for item in EXTERNAL_PLUGIN_SET.members()]
+    config.api.external_sources = [item for item in config.api.external_sources if item.built_in is True]
+    config.api.external_sources.extend(plugin_list)
+
+
+def setup_job(sched: BaseScheduler):
+    """Sets the scheduler instance to the one provided, and refreshes the external sources.
+
+    Arguments:
+        sched: The scheduler instance to set.
+    """
+    global __scheduler_instance
+    __scheduler_instance = sched
+    sched.add_job(update_external_source_list, "interval", minutes=1)
+    logger.debug("Plugin job setup complete.")

clue/error.py

@@ -0,0 +1,129 @@
+from http.client import HTTPException
+from sys import exc_info
+from traceback import format_tb
+from typing import Union
+
+from flask import Blueprint, request
+from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, Unauthorized
+
+from clue.api import bad_request, forbidden, internal_error, not_found, unauthorized
+from clue.common.exceptions import (
+    AccessDeniedException,
+    AuthenticationException,
+    ClueException,
+)
+from clue.common.logging import get_logger, log_with_traceback
+from clue.common.logging.audit import AUDIT
+from clue.config import config
+
+errors = Blueprint("errors", __name__)
+
+logger = get_logger(__file__)
+
+
+######################################
+# Custom Error page
+@errors.app_errorhandler(400)
+def handle_400(e: Union[HTTPException, ClueException]):
+    """Handles HTTP 400 Bad Request errors.
+
+    If the error is not an instance of BadRequest, the string representation of that error will be included in the
+    response.
+
+    Arguments:
+        e: The error to handle.
+
+    Returns:
+        A Response object representing a Bad Request HTTP error.
+    """
+    if isinstance(e, BadRequest):
+        error_message = "No data block provided or data block not in JSON format.'"
+    else:
+        error_message = str(e)
+    return bad_request(err=error_message)
+
+
+@errors.app_errorhandler(401)
+def handle_401(e: Union[HTTPException, ClueException]):
+    """Handles HTTP 401 Unauthorized errors.
+
+    If the error is not an instance of Unauthorized, the string representation of that error will be included in the
+    response. It will also clear the XSRF-TOKEN.
+
+    Arguments:
+        e: The error to handle.
+
+    Returns:
+        A Response object representing an Unauthorized HTTP error, also containing the oauth_providers data.
+    """
+    if isinstance(e, Unauthorized):
+        msg = e.description
+    else:
+        msg = str(e)
+
+    data = {"oauth_providers": [name for name in config.auth.oauth.providers.keys()]}
+    res = unauthorized(data, err=msg)
+    res.set_cookie("XSRF-TOKEN", "", max_age=0)
+    return res
+
+
+@errors.app_errorhandler(403)
+def handle_403(e: Union[HTTPException, ClueException]):
+    """Handles HTTP 403 Forbidden errors.
+
+    If the error is not an instance of Forbidden, the string representation of that error will be included in the
+    response. If the AUDIT config is enabled, this request will be logged.
+
+    Arguments:
+        e: The error to handle.
+
+    Returns:
+        A Response object representing a Forbidden HTTP error.
+    """
+    if isinstance(e, Forbidden):
+        error_message = e.description
+    else:
+        error_message = str(e)
+
+    trace = exc_info()[2]
+    if AUDIT:
+        uname = "(None)"
+        ip = request.remote_addr
+
+        log_with_traceback(trace, f"Access Denied. (U:{uname} - IP:{ip}) [{error_message}]", audit=True)
+
+    return forbidden(err=f"Access Denied ({request.path}) [{error_message}]")
+
+
+@errors.app_errorhandler(404)
+def handle_404(_):
+    """Handles HTTP 404 Not Found errors."""
+    return not_found(err=f"Api does not exist ({request.path})")
+
+
+@errors.app_errorhandler(500)
+def handle_500(e: InternalServerError):
+    """Handles HTTP 500 Internal Server errors.
+
+    If the original_exception of e is an AccessDeniedException or AuthenticationException, this redirects to the 403 or
+    401 error handlers, otherwise it logs it and returns a formatted error message.
+
+    Arguments:
+        e: The error to handle.
+
+    Returns:
+        A Response object representing an Internal Server HTTP error.
+    """
+    if isinstance(e.original_exception, AccessDeniedException):
+        return handle_403(e.original_exception)
+
+    if isinstance(e.original_exception, AuthenticationException):
+        return handle_401(e.original_exception)
+
+    oe = e.original_exception or e
+
+    trace = exc_info()[2]
+    log_with_traceback(trace, "Exception", is_exception=True)
+
+    message = "".join(["\n"] + format_tb(exc_info()[2]) + ["%s: %s\n" % (oe.__class__.__name__, str(oe))]).rstrip("\n")
+    return internal_error(err=message)