clue-api 1.0.0.dev7__py3-none-any.whl

clue/remote/datatypes/set.py

@@ -0,0 +1,96 @@
+import json
+import time
+
+from clue.remote.datatypes import get_client, retry_call
+
+_drop_card_script = """
+local set_name = ARGV[1]
+local key = ARGV[2]
+
+redis.call('srem', set_name, key)
+return redis.call('scard', set_name)
+"""
+
+_limited_add = """
+local set_name = KEYS[1]
+local key = ARGV[1]
+local limit = tonumber(ARGV[2])
+
+if redis.call('scard', set_name) < limit then
+    redis.call('sadd', set_name, key)
+    return true
+end
+return false
+"""
+
+
+class Set(object):
+    def __init__(self, name, host=None, port=None):
+        self.c = get_client(host, port, False)
+        self.name = name
+        self._drop_card = self.c.register_script(_drop_card_script)
+        self._limited_add = self.c.register_script(_limited_add)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.delete()
+
+    def add(self, *values):
+        return retry_call(self.c.sadd, self.name, *[json.dumps(v) for v in values])
+
+    def limited_add(self, value, size_limit):
+        """Add a single value to the set, but only if that wouldn't make the set grow past a given size."""
+        return retry_call(self._limited_add, keys=[self.name], args=[json.dumps(value), size_limit])
+
+    def exist(self, value):
+        return retry_call(self.c.sismember, self.name, json.dumps(value))
+
+    def length(self):
+        return retry_call(self.c.scard, self.name)
+
+    def members(self):
+        return [json.loads(s) for s in retry_call(self.c.smembers, self.name)]
+
+    def remove(self, *values):
+        return retry_call(self.c.srem, self.name, *[json.dumps(v) for v in values])
+
+    def drop(self, value):
+        return retry_call(self._drop_card, args=[value])
+
+    def random(self, num=None):
+        ret_val = retry_call(self.c.srandmember, self.name, num)
+        if isinstance(ret_val, list):
+            return [json.loads(s) for s in ret_val]
+        else:
+            return json.loads(ret_val)
+
+    def pop(self):
+        data = retry_call(self.c.spop, self.name)
+        return json.loads(data) if data else None
+
+    def pop_all(self):
+        return [json.loads(s) for s in retry_call(self.c.spop, self.name, self.length())]
+
+    def delete(self):
+        retry_call(self.c.delete, self.name)
+
+
+class ExpiringSet(Set):
+    def __init__(self, name, ttl=86400, host=None, port=None):
+        super(ExpiringSet, self).__init__(name, host, port)
+        self.ttl = ttl
+        self.last_expire_time = 0
+
+    def _conditional_expire(self):
+        if self.ttl:
+            ctime = time.time()
+            if ctime > self.last_expire_time + (self.ttl / 2):
+                retry_call(self.c.expire, self.name, self.ttl)
+                self.last_expire_time = ctime
+
+    def add(self, *values):
+        rval = super(ExpiringSet, self).add(*values)
+        self._conditional_expire()
+        return rval

clue/remote/datatypes/user_quota_tracker.py

@@ -0,0 +1,54 @@
+import redis
+
+from clue.remote.datatypes import get_client, retry_call
+
+begin_script = """
+local t = redis.call('time')
+local key = tonumber(t[1] .. string.format("%06d", t[2]))
+
+local name = ARGV[1]
+local max = tonumber(ARGV[2])
+local timeout = tonumber(ARGV[3] .. "000000")
+
+redis.call('zremrangebyscore', name, 0, key - timeout)
+if redis.call('zcard', name) < max then
+    redis.call('zadd', name, key, key)
+    return true
+else
+    return false
+end
+"""
+
+
+class UserQuotaTracker(object):
+    def __init__(self, prefix, timeout=120, redis=None, host=None, port=None, private=False):
+        self.c = redis or get_client(host, port, private)
+        self.bs = self.c.register_script(begin_script)
+        self.prefix = prefix
+        self.timeout = timeout
+
+    def _queue_name(self, user):
+        return f"{self.prefix}-{user}"
+
+    def begin(self, user: str, max_quota: int) -> bool:
+        try:
+            return retry_call(self.bs, args=[self._queue_name(user), max_quota, self.timeout]) == 1
+        except redis.exceptions.ResponseError as er:
+            # TODO: This is a failsafe for upgrade purposes could be removed in a future version
+            if "WRONGTYPE" in str(er):
+                retry_call(self.c.delete, self._queue_name(user))
+                return retry_call(self.bs, args=[self._queue_name(user), max_quota, self.timeout]) == 1
+            else:
+                raise
+
+    def end(self, user: str):
+        """When only one item is requested, blocking it is possible."""
+        try:
+            retry_call(self.c.zpopmin, self._queue_name(user))
+        except redis.exceptions.ResponseError as er:
+            # TODO: This is a failsafe for upgrade purposes could be removed in a future version
+            if "WRONGTYPE" in str(er):
+                retry_call(self.c.delete, self._queue_name(user))
+                retry_call(self.c.zpopmin, self._queue_name(user))
+            else:
+                raise

clue/security/__init__.py

@@ -0,0 +1,211 @@
+import functools
+from typing import Callable, Optional
+
+import elasticapm
+import requests
+from flask import request
+from jwt import ExpiredSignatureError
+from prometheus_client import Counter
+
+import clue.services.auth_service as auth_service
+from clue.api import bad_request, forbidden, internal_error, not_found, unauthorized
+from clue.common.exceptions import (
+    AccessDeniedException,
+    AuthenticationException,
+    ClueAttributeError,
+    ClueNotImplementedError,
+    ClueRuntimeError,
+    InvalidDataException,
+    NotFoundException,
+)
+from clue.common.forge import APP_NAME
+from clue.common.logging import get_logger
+from clue.common.logging.audit import audit
+from clue.config import AUDIT, config
+
+logger = get_logger(__file__)
+
+SUCCESSFUL_ATTEMPTS = Counter(
+    f"{APP_NAME.replace('-', '_')}_auth_success_total",
+    "Successful Authentication Attempts",
+)
+
+FAILED_ATTEMPTS = Counter(
+    f"{APP_NAME.replace('-', '_')}_auth_fail_total",
+    "Failed Authentication Attempts",
+    ["status"],
+)
+
+XSRF_ENABLED = True
+
+
+####################################
+# API Helper func and decorators
+# noinspection PyPep8Naming
+class api_login(object):  # noqa: N801
+    """Adds authentication to an endpoint"""
+
+    def __init__(
+        self,  # noqa: ANN101
+        # TODO: Fix type parsing and checks
+        # required_type: Optional[list[str]] = None,
+        username_key: str = "username",
+        audit: bool = True,
+        required_priv: Optional[list[str]] = None,
+        required_method: Optional[list[str]] = None,
+        check_xsrf_token: bool = XSRF_ENABLED,
+    ):
+        if required_priv is None:
+            required_priv = ["R", "W"]
+
+        # TODO: Fix type parsing and checks
+        # if required_type is None:
+        #     required_type = ["admin", "user"]
+
+        required_method_set: set[str]
+        if required_method is None:
+            required_method_set = {"userpass", "apikey", "internal", "oauth"}
+        else:
+            required_method_set = set(required_method)
+
+        if len(required_method_set - {"userpass", "apikey", "internal", "oauth"}) > 0:
+            raise ClueAttributeError("required_method must be a subset of {userpass, apikey, internal, oauth}")
+
+        # TODO: Fix type parsing and checks
+        # self.required_type = required_type
+        self.audit = audit and AUDIT
+        self.required_priv = required_priv
+        self.required_method = required_method_set
+        self.username_key = username_key
+        self.check_xsrf_token = check_xsrf_token
+
+    def __call__(self, func: Callable) -> Callable:  # noqa: ANN101, C901
+        """Wraps any function calls with authentication logic that uses either userpass, apikey, internal or oauth.
+
+        Args:
+            func (Callable): The function to wrap with auth.
+
+        Raises:
+            AuthenticationException: Raised whenever there's an actual problem with the provided authentication.
+            InvalidDataException: Raised whenever data is incorrectly formatted.
+            ClueRuntimeError: Raised whenever there is a connection error with the oauth provider
+            AccessDeniedException: Raised whenever the authentication is valid but the authenticated identity doesn't
+                have the required access
+
+        Returns:
+            _type_: _description_
+        """
+
+        @functools.wraps(func)
+        def base(*args, **kwargs):  # noqa: C901
+            try:
+                # All authorization (except impersonation) must go through the Authorization header, in one of
+                # four formats:
+                # 1. Basic user/pass authentication
+                #       Authorization: Basic username:password (but in base64)
+                # 2. Basic user/apikey authentication
+                #       Authorization: Basic username:keyname:keydata (but in base64)
+                # 3. Bearer internal token authentication (obtained from the login endpoint)
+                #       Authorization: Bearer username:token
+                # 4. Bearer OAuth authentication (obtained from external authentication provider i.e. azure, keycloak)
+                #       Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjMifQ (example)
+                authorization = request.headers.get("Authorization", None)
+                if not authorization:
+                    raise AuthenticationException("No Authorization header present")
+                elif " " not in authorization or len(authorization.split(" ")) > 2:
+                    raise InvalidDataException("Incorrectly formatted Authorization header")
+
+                logger.debug("Authenticating user for path %s", request.path)
+
+                [auth_type, data] = authorization.split(" ")
+
+                user = None
+                if auth_type == "Basic" and len(self.required_method & {"userpass", "apikey"}) > 0:
+                    # Authenticate case (1) and (2) above
+                    user, priv = auth_service.basic_auth(
+                        data,
+                        skip_apikey="apikey" not in self.required_method,
+                        skip_password="userpass" not in self.required_method,
+                    )
+                elif auth_type == "Bearer" and len(self.required_method & {"internal", "oauth"}) > 0:
+                    # Authenticate case (3) and (4) above
+                    try:
+                        user, priv = auth_service.bearer_auth(
+                            data,
+                            skip_jwt="oauth" not in self.required_method,
+                            skip_internal="internal" not in self.required_method,
+                        )
+                    except ExpiredSignatureError as e:
+                        raise AuthenticationException("Token Expired") from e
+                    except (requests.exceptions.ConnectionError, ConnectionError) as e:
+                        logger.exception("Failed to connect to OAuth Provider:")
+                        raise ClueRuntimeError("Failed to connect to OAuth Provider") from e
+                else:
+                    raise InvalidDataException("Not a valid authentication type for this endpoint.")
+
+                if not user:
+                    raise AuthenticationException("No authenticated user found")
+
+                # Ensure that the provided api key allows access to this API
+                if not priv or not set(self.required_priv) & set(priv):
+                    raise AccessDeniedException("You do not have access to this API.")
+
+                # Make sure the user has the correct type for this endpoint
+                # TODO: Fix type parsing and checks
+                # if not set(self.required_type) & set(user["type"]):
+                #     logger.warning(
+                #         f"{user['uname']} is missing one of the types: {', '.join(self.required_type)}. "
+                #         "Cannot access {request.path}"
+                #     )
+                #     raise AccessDeniedException(
+                #         f"{request.path} requires one of the following user types: {', '.join(self.required_type)}"
+                #     )
+
+                ip = request.headers.get("X-Forwarded-For", request.remote_addr)
+                logger.info(f"Logged in as {user['uname']} from {ip}")
+
+                # If auditing is enabled, write this successful access to the audit logs
+                if self.audit:
+                    audit(
+                        args,
+                        kwargs,
+                        user,
+                        func,
+                    )
+            except (InvalidDataException, ClueNotImplementedError) as e:
+                FAILED_ATTEMPTS.labels("400").inc()
+                return bad_request(err=e.message)
+            except AuthenticationException as e:
+                FAILED_ATTEMPTS.labels("401").inc()
+                return unauthorized(err=e.message)
+            except AccessDeniedException as e:
+                FAILED_ATTEMPTS.labels("403").inc()
+                return forbidden(err=e.message)
+            except NotFoundException:
+                FAILED_ATTEMPTS.labels("404").inc()
+                return not_found()
+            except ClueRuntimeError as e:
+                FAILED_ATTEMPTS.labels("500").inc()
+                return internal_error(err=e.message)
+
+            if config.core.metrics.apm_server.server_url is not None:
+                elasticapm.set_user_context(
+                    username=user.get("name", None),
+                    email=user.get("email", None),
+                    user_id=user.get("uname", None),
+                )
+
+            # Save user data in kwargs for future reference in the wrapped method
+            kwargs["user"] = user
+
+            SUCCESSFUL_ATTEMPTS.inc()
+            return func(*args, **kwargs)
+
+        base.protected = True
+        # TODO: Fix type parsing and checks
+        # base.required_type = self.required_type
+        base.audit = self.audit
+        base.required_priv = self.required_priv
+        base.required_method = self.required_method
+        base.check_xsrf_token = self.check_xsrf_token
+        return base

clue/security/obo.py

@@ -0,0 +1,95 @@
+from datetime import datetime
+from typing import Optional
+
+# from hogwarts.auth.vault.exceptions import VaultRequestException
+# from hogwarts.auth.vault.vault_client import VaultClient
+from clue.common.exceptions import InvalidDataException
+from clue.common.logging import get_logger
+from clue.config import config, get_redis
+from clue.remote.datatypes.set import ExpiringSet
+from clue.security.utils import decode_jwt_payload
+
+logger = get_logger(__file__)
+
+
+def _get_obo_token_store(service: str, user: str) -> ExpiringSet:
+    """Get an expiring redis set in which to add a token
+
+    Args:
+        user (str): The user the token corresponds to
+
+    Returns:
+        ExpiringSet: The set in which we'll store the token
+    """
+    return ExpiringSet(f"{service}_token_{user}", host=get_redis(), ttl=60 * 5)
+
+
+def _get_token_raw(service: str, user: str) -> Optional[str]:
+    token_store = _get_obo_token_store(service, user)
+
+    if token_store.length() > 0:
+        return token_store.random(1)[0]
+
+    return None
+
+
+def get_obo_token(service: str, access_token: str, user: str, force_refresh: bool = False):
+    """Gets an On-Behalf-Of token from either the Redis cache or from the Vault API.
+
+    Args:
+        service (str): The target application we want a token for.
+        access_token (str): The access token we want to use for the exchange.
+        user (str): The name of the user.
+        force_refresh (bool, optional): Allows to skip the Redis cache and get a new token. Defaults to False.
+
+    Raises:
+        InvalidDataException: Raised whenever an invalid OBO target is provided.
+
+    Returns:
+        Optional[str]: The access token for the targeted application.
+    """
+    if service not in config.api.obo_targets:
+        raise InvalidDataException("Not a valid OBO target")
+
+    # For testing purposes, we special-case test-obo
+    if service == "test-obo":
+        return access_token
+
+    try:
+        obo_access_token: str | None = None
+
+        if not force_refresh:
+            obo_access_token = _get_token_raw(service, user)
+
+        if obo_access_token is not None:
+            expiry = datetime.fromtimestamp(decode_jwt_payload(obo_access_token)["exp"])
+
+            if expiry < datetime.now():
+                logger.warning("Cached token has expired")
+                obo_access_token = None
+
+        if obo_access_token is None:
+            logger.info(f"Fetching OBO token for user {user} to service {service}")
+
+            logger.debug("Contacting vault for new OBO token")
+            # vault_client = VaultClient(url=config.api.vault_url)
+            # obo_access_token, _ = vault_client.on_behalf_of(
+            #     config.api.obo_targets[service].scope,
+            #     access_token,
+            #     token_client_name=APP_NAME.replace("-dev", ""),
+            # )
+            obo_access_token = None
+
+            if obo_access_token:
+                service_token_store = _get_obo_token_store(service, user)
+                service_token_store.pop_all()
+                service_token_store.add(obo_access_token)
+            else:
+                logger.error("Vault OBO failed, no token received.")
+        else:
+            logger.debug("Using cached OBO token")
+
+        return obo_access_token
+    except Exception:
+        # except VaultRequestException:
+        logger.exception("VaultRequestException on OBO:")

clue/security/utils.py

@@ -0,0 +1,34 @@
+import base64
+import json
+import os
+from typing import Any
+
+UPPERCASE = r"[A-Z]"
+LOWERCASE = r"[a-z]"
+NUMBER = r"[0-9]"
+SPECIAL = r'[ !#$@%&\'()*+,-./[\\\]^_`{|}~"]'
+PASS_BASIC = (
+    [chr(x + 65) for x in range(26)]
+    + [chr(x + 97) for x in range(26)]
+    + [str(x) for x in range(10)]
+    + ["!", "@", "$", "^", "?", "&", "*", "(", ")"]
+)
+
+
+def generate_random_secret(length: int = 25) -> str:
+    """Generate a random secret
+
+    Args:
+        length (int, optional): The length of the secret. Defaults to 25.
+
+    Returns:
+        str: The random secret
+    """
+    return base64.b32encode(os.urandom(length)).decode("UTF-8")
+
+
+def decode_jwt_payload(jwt: str) -> dict[str, Any]:
+    "Decode a JWT payload. DOES NOT VALIDATE THE JWT, DO NOT USE THIS TO VALIDATE THE TOKEN."
+    payload = jwt.split(".")[1]
+
+    return json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)).decode())

clue/services/action_service.py

@@ -0,0 +1,186 @@
+from typing import Any, Optional
+from urllib.parse import urljoin
+
+import elasticapm
+import requests
+from flask import request
+from pydantic import TypeAdapter, ValidationError
+from requests import JSONDecodeError, exceptions
+
+from clue.common.exceptions import ClueException, NotFoundException
+from clue.common.logging import get_logger
+from clue.config import CLASSIFICATION, config
+from clue.helper.headers import generate_headers
+from clue.models.actions import ActionResult, ActionSpec
+from clue.models.config import ExternalSource
+from clue.services import auth_service
+
+logger = get_logger(__file__)
+
+
+def get_supported_actions(
+    source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
+) -> dict[str, ActionSpec]:
+    """Gets all supported actions for a source
+
+    Args:
+        source_url (str): The URL of the source
+        access_token (Optional[str], optional): The access token to use, if necessary. Defaults to None.
+
+    Returns:
+        dict[str, ActionSpec]: A dict of each action and their schema
+    """
+    logger.info("Fetching actions for source %s", source.name)
+
+    url = urljoin(source.url, "actions/")
+
+    obo_access_token = None
+    if access_token:
+        obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
+
+        if error:
+            logger.error("%s: %s", source.name, error)
+            return {}
+
+    headers = generate_headers(obo_access_token or access_token, access_token if obo_access_token else None)
+
+    with elasticapm.capture_span(f"GET {url}", span_type="http"):
+        try:
+            rsp = requests.get(url, headers=headers, timeout=10.0)
+            result = rsp.json()
+
+            if not rsp.ok:
+                err = result["api_error_message"]
+                logger.error(f"Error from upstream server: {rsp.status_code=}, {err=}")
+
+            return TypeAdapter(dict[str, ActionSpec]).validate_python(result["api_response"])
+        except exceptions.ConnectionError:
+            # any errors are logged and no result is saved to local cache to enable retry on next query
+            logger.exception("Unable to connect: %s", url)
+            return {}
+        except (requests.exceptions.JSONDecodeError, KeyError, JSONDecodeError):
+            logger.exception("External API did not return expected format. Full data:\n\n%s\n\nStack Trace:", rsp.text)
+            return {}
+        except ValidationError:
+            logger.exception("ValidationError in response from %s:\n%s", source.url)
+            return {}
+        except Exception:
+            logger.exception("Unknown exception occurred on action fetching:")
+            return {}
+
+
+def all_supported_actions(user: dict[str, Any], access_token: Optional[str] = None) -> dict[str, ActionSpec]:
+    """Gets all supported actions for all sources
+
+    Args:
+        access_token (Optional[str], optional): The access token to use, if necessary. Defaults to None.
+
+    Returns:
+        dict[str, ActionSpec]: A dict of all actions and their matching schema
+    """
+    all_actions: dict[str, ActionSpec] = {}
+
+    for source in config.api.external_sources:
+        supported_actions = get_supported_actions(source, user, access_token=access_token)
+        total_actions = 0
+        for key, action in supported_actions.items():
+            total_actions += 1
+            all_actions[f"{source.name}.{key}"] = action
+        logger.debug("Plugin %s exposes %s action(s)", source.name, total_actions)
+
+    return all_actions
+
+
+def get_plugins_supported_actions(user: dict[str, Any]) -> dict[str, ActionSpec]:
+    """Return the supported actions of each external service, filtered to what the user has access to."""
+    available_actions: dict[str, ActionSpec] = {}
+
+    access_token = request.headers.get("Authorization", type=str)
+    if access_token:
+        access_token = access_token.split(" ")[1]
+
+    all_actions = all_supported_actions(
+        user,
+        access_token=access_token,
+    )
+
+    logger.info("Fetching actions for classification %s", user["classification"])
+
+    for action_id, action in all_actions.items():
+        # Validate if the user is allow to even see the source
+        if user and not CLASSIFICATION.is_accessible(user["classification"], action.classification):
+            logger.info(
+                "Not including actions from source %s at classification %s", action.name, user["classification"]
+            )
+            continue
+
+        # user can view source, now filter types user cannot see
+        available_actions[action_id] = action
+
+    logger.info("%s actions are available for user %s", len(available_actions), user["uname"])
+
+    return available_actions
+
+
+def execute_action(plugin_id: str, action_id: str, user: dict[str, Any]) -> ActionResult:
+    """Executes a specified action.
+
+    Args:
+        plugin_id (str): The ID of the plugin.
+        action_id (str): The ID of the action to run.
+        user (dict[str, Any]): The user dict of the user running the action.
+
+    Raises:
+        NotFoundException: Raised whenever the plugin or the action doesn't exist.
+        ClueException: Raised whenever an error is returned by the plugin endpoint.
+
+    Returns:
+        ActionResult: The result of the action.
+    """
+    plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
+
+    if not plugin:
+        raise NotFoundException(f"Plugin {plugin_id} does not exist.")
+
+    access_token = request.headers.get("Authorization", type=str)
+    if access_token:
+        access_token = access_token.split(" ")[1]
+
+    obo_access_token = None
+    if access_token:
+        obo_access_token, error = auth_service.check_obo(plugin, access_token, user["uname"])
+
+        if error:
+            logger.error("%s: %s", plugin.name, error)
+            return ActionResult(outcome="failure", summary="Invalid token provided for this enrichment.")
+
+    headers = generate_headers(obo_access_token or access_token, access_token if obo_access_token else None)
+
+    if request.content_type == "application/json":
+        parameters = request.json
+    else:
+        # TODO: Pass parameters via urlencode?
+        parameters = {}
+
+    try:
+        req_url = urljoin(plugin.url, f"actions/{action_id}")
+        logger.debug("Executing action %s for user %s", req_url, user["uname"])
+
+        response = requests.post(
+            req_url,
+            json=parameters,
+            headers=headers,
+            timeout=request.args.get("max_timeout", plugin.default_timeout, type=float),
+        )
+
+        result = response.json()
+
+        if not response.ok:
+            raise ClueException(result["api_error_message"])
+
+        return ActionResult.model_validate(result["api_response"])
+    except (JSONDecodeError, exceptions.ConnectionError) as err:
+        logger.exception(f"Something went wrong when retrieving the result from plugin '{plugin_id}'")
+        raise ClueException(
+            f"Something went wrong when retrieving the result from plugin '{plugin_id}': {err.__class__.__name__}."
+        )