clue-api 1.0.0.dev7__py3-none-any.whl

clue/services/auth_service.py

@@ -0,0 +1,348 @@
+import base64
+import hashlib
+from typing import Any, Optional, Union
+
+import elasticapm
+from flask import request
+
+from clue.common.exceptions import (
+    AccessDeniedException,
+    AuthenticationException,
+    ClueException,
+    ClueNotImplementedError,
+    InvalidDataException,
+)
+from clue.common.logging import get_logger
+from clue.config import config, get_redis
+from clue.models.config import ExternalSource
+from clue.remote.datatypes.set import ExpiringSet
+from clue.security.obo import get_obo_token
+from clue.security.utils import decode_jwt_payload, generate_random_secret
+from clue.services import jwt_service, user_service
+
+logger = get_logger(__file__)
+
+redis_config: dict[str, Union[str, int]] = {
+    "host": config.core.redis.host,
+    "port": config.core.redis.port,
+}
+
+redis = get_redis()
+
+
+def _get_token_store(user: str) -> ExpiringSet:
+    """Get an expiring redis set in which to add a token
+
+    Args:
+        user (str): The user the token corresponds to
+
+    Returns:
+        ExpiringSet: The set in which we'll store the token
+    """
+    return ExpiringSet(f"token_{user}", host=redis, ttl=60 * 60)  # 1 Hour expiry
+
+
+def _get_priv_store(user: str, token: str) -> ExpiringSet:
+    """Get an expiring redis set in which to add the privileges
+
+    Args:
+        user (str): The user the token corresponds to
+        token (str): The token the privileges correspond to
+
+    Returns:
+        ExpiringSet: The set in which we'll store the privileges
+    """
+    return ExpiringSet(
+        # For security reasons, we won't save the whole token in redis. Just in case :)
+        f"token_priv_{user}_{token[:10]}",
+        host=redis,
+        # 1 Hour expiry
+        ttl=60 * 60,
+    )
+
+
+def create_token(user: str, priv: list[str]) -> str:
+    """Generate a new token associated with the given user with the given privileges
+
+    Args:
+        user (str): The user to create the token as
+        priv (list[str]): The privileges to give the token
+
+    Returns:
+        str: The new token
+    """
+    token = hashlib.sha256(str(generate_random_secret()).encode("utf-8", errors="replace")).hexdigest()
+
+    _get_token_store(user).add(token)
+    priv_store = _get_priv_store(user, token)
+    priv_store.pop_all()
+    priv_store.add(",".join(priv))
+
+    return token
+
+
+def check_token(user: str, token: str) -> Optional[list[str]]:
+    """Check if a token exists, and return its list of privileges
+
+    Args:
+        user (str): The user corresponding to the token to check
+        token (str): The token
+
+    Returns:
+        Optional[list[str]]: The list of privileges associated with the token
+    """
+    if _get_token_store(user).exist(token):
+        members = _get_priv_store(user, token).members()
+        if len(members) > 0:
+            priv_str = members[0]
+            return priv_str.split(",")
+
+    return None
+
+
+def validate_token(username: str, token: str) -> Optional[list[str]]:
+    """This function identifies the user via the internal token functionality
+
+    Args:
+        username (str): The username corresponding to the provided token
+        token (str): The token generated by our API to check for
+
+    Raises:
+        AuthenticationException: Invalid token
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    if token:
+        priv = check_token(username, token)
+        if priv:
+            return priv
+
+        raise AuthenticationException("Invalid token")
+
+    return None
+
+
+@elasticapm.capture_span(span_type="authentication")
+def bearer_auth(
+    data: str, skip_jwt: bool = False, skip_internal: bool = False
+) -> tuple[Optional[dict[str, Any]], Optional[list[str]]]:
+    """This function handles Bearer type Authorization headers.
+
+    Args:
+        data (str): The corresponding data in the Authorization header.
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    if "." in data:
+        if not skip_jwt:
+            try:
+                jwt_data = jwt_service.decode(data, validate_audience=True)
+            except ClueException as e:
+                logger.exception("Exception when decoding JWT:")
+                raise AuthenticationException(
+                    "Something went wrong when decoding your key. Please reauthenticate.",
+                    cause=e,
+                )
+
+            if not jwt_data:
+                logger.error("Invalid JWT provided.")
+                raise AuthenticationException("Invalid JWT, please reauthenticate.")
+
+            logger.debug("User successfully authenticated using JWT.")
+
+            cur_user = user_service.parse_user_data(jwt_data, jwt_service.get_provider(data))
+
+            return cur_user, ["R", "W"]
+        else:
+            raise InvalidDataException("Not a valid authentication type for this endpoint.")
+    else:
+        if not skip_internal:
+            raise ClueNotImplementedError("Internal bearer auth is not yet supported.")
+        else:
+            raise InvalidDataException("Not a valid authentication type for this endpoint.")
+
+
+@elasticapm.capture_span(span_type="authentication")
+def validate_apikey(name: str, apikey: str) -> tuple[Optional[dict[str, Any]], Optional[list[str]]]:
+    """This function identifies the user via the internal API key functionality.
+
+    Args:
+        name (str): Name of the APIKey to check against
+        apikey (str): The apikey used to authenticate as the user
+
+    Raises:
+        AccessDeniedException: Api Key authentication was disabled, or the api was not valid for impersonation,
+                               or it was an impersonation api key incorrectly provided in the Authorization header.
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    if not config.auth.allow_apikeys:
+        raise AccessDeniedException("API Key authentication disabled")
+
+    if name and apikey:
+        config_apikey = config.auth.apikeys.get(name, None)
+        if not config_apikey:
+            raise AccessDeniedException("API Key does not exist")
+
+        if config_apikey != apikey:
+            raise AccessDeniedException("Invalid API key")
+
+        uname = request.headers.get("X-USERID", None)
+        classification = request.headers.get("X-CLASSIFICATION", None)
+        user_name = request.headers.get("X-USERNAME", None)
+        email = request.headers.get("X-EMAIL", None)
+        if not uname or not classification:
+            raise AccessDeniedException(
+                "You must also provide X-USERID and X-CLASSIFICATION headers along with you API key."
+            )
+
+        return {"uname": uname, "name": user_name, "classification": classification, "email": email}, ["R", "W"]
+    else:
+        raise AccessDeniedException("You must provide your API key in the proper format in the Authorization header.")
+
+
+def validate_userpass(username: str, password: str) -> tuple[Optional[dict[str, Any]], Optional[list[str]]]:
+    """This function identifies the user via the user/pass functionality. (NOT IMPLEMENTED)
+
+    Args:
+        username (str): The username corresponding to the provided password
+        password (str): The password used to authenticate as the user
+
+    Raises:
+        AccessDeniedException: Username/Password authentication is currently disabled
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    raise ClueNotImplementedError("API key auth is not yet supported.")
+
+    # if config.auth.internal.enabled and username and password:
+    #     raise ClueNotImplementedError("API key auth is not yet supported.")
+    # else:
+    #     raise AccessDeniedException("Username/Password authentication disabled")
+
+
+def decode_b64(b64_str: str) -> str:
+    """Decode a base64 string into plain text.
+
+    Args:
+        b64_str (str): The base64 string
+
+    Raises:
+        InvalidDataException: The data was not base64.
+
+    Returns:
+        str: A plain text representation of the data.
+    """
+    try:
+        return base64.b64decode(b64_str).decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise InvalidDataException("Basic authentication data must be base64 encoded") from e
+
+
+@elasticapm.capture_span(span_type="authentication")
+def basic_auth(
+    data: str, is_base64: bool = True, skip_apikey: bool = False, skip_password: bool = False
+) -> tuple[Optional[dict[str, Any]], Optional[list[str]]]:
+    """This function handles Basic type Authorization headers.
+
+    Args:
+        data (str): The corresponding data in the Authorization header.
+        is_base64 (bool, optional): Whether the provided data is base64 encoded. Defaults to True.
+        skip_apikey (bool, optional): Whether to skip apikey validation. Defaults to False.
+        skip_password (bool, optional): Whether to skip password validation. Defaults to False.
+
+    Raises:
+        AuthenticationException: The login information is invalid, or the maximum password retry for the account
+                                 has been reached.
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    key_pair = decode_b64(data) if is_base64 else data
+
+    [username, data] = key_pair.split(":", maxsplit=1)
+
+    validated_user = None
+    if not skip_apikey:
+        validated_user, priv = validate_apikey(username, data)
+
+    # Bruteforce protection
+    # auth_fail_queue: NamedQueue = NamedQueue(f"ui-failed-{username}", **redis_config)  # type: ignore
+    # if auth_fail_queue.length() >= config.auth.internal.max_failures:
+    #     # Failed 'max_failures' times, stop trying... This will timeout in 'failure_ttl' seconds
+    #     raise AuthenticationException(
+    #         "Maximum password retry of {retry} was reached. "
+    #         "This account is locked for the next {ttl} "
+    #         "seconds...".format(
+    #             retry=config.auth.internal.max_failures,
+    #             ttl=config.auth.internal.failure_ttl,
+    #         )
+    #     )
+
+    if not validated_user and not skip_password:
+        validated_user, priv = validate_userpass(username, data)
+
+    if not validated_user:
+        # auth_fail_queue.push(
+        #     {
+        #         "remote_addr": request.remote_addr,
+        #         "host": request.host,
+        #         "full_path": request.full_path,
+        #     }
+        # )
+        raise AuthenticationException("Invalid login information")
+
+    return validated_user, priv
+
+
+def extract_audience(access_token: str) -> list[str]:
+    "Extract the audience from an encoded JWT."
+    audience: list[str] | str | None = decode_jwt_payload(access_token).get("aud", None)
+
+    if not audience:
+        return []
+
+    return [audience] if not isinstance(audience, list) else audience
+
+
+# TODO: sa-clue support
+def check_obo(source: ExternalSource, access_token: str, username: str) -> tuple[Optional[str], Optional[str]]:
+    """Checks whether a token's audience matches the source, and if it doesn't, tries to get an OBO token for the source
+
+    Args:
+        source (ExternalSource): The source for which we want an access token.
+        access_token (str): The access token to verify.
+        username (str): The name of the user.
+
+    Returns:
+        tuple[Optional[str], Optional[str]]: A tuple with either the valid access token in the first arg, or the error
+            message in the second arg.
+    """
+    obo_access_token: Optional[str] = None
+    if source.obo_target:
+        logger.debug("Source %s requires OBO to %s", source.name, source.obo_target)
+
+        if "." not in access_token:
+            logger.warning("JWT not provided, using service account.")
+            if not (sa_token := jwt_service.fetch_sa_token()):
+                return None, None
+
+            access_token = sa_token
+
+        audience = extract_audience(access_token)
+
+        # Check if this is a standard clue token
+        if jwt_service.get_audience(jwt_service.get_provider(access_token)) in audience:
+            obo_access_token = get_obo_token(source.obo_target, access_token, username)
+            if not obo_access_token:
+                return None, "An error occurred when OBOing token"
+
+        # Check if the scope already matches (could happen in cases like howler -> clue -> howler)
+        elif config.api.obo_targets[source.obo_target].scope.split("/")[0] not in audience:
+            return None, "Invalid token for this enrichment"
+
+    return obo_access_token, None

clue/services/config_service.py

@@ -0,0 +1,38 @@
+from clue.common.str_utils import default_string_value
+from clue.config import CLASSIFICATION, config, get_branch, get_commit, get_version
+from clue.helper.discover import get_apps_list
+
+classification_definition = CLASSIFICATION.get_parsed_classification_definition()
+
+apps = get_apps_list()
+
+
+def get_configuration():
+    """Get system configration data for the Clue API
+
+    Args:
+        user (User): The user making the request
+    """
+    return {
+        "configuration": {
+            "auth": {
+                "oauth_providers": [
+                    name
+                    for name, p in config.auth.oauth.providers.items()
+                    if default_string_value(p.client_secret, env_name=f"{name.upper()}_CLIENT_SECRET")
+                ],
+                # "internal": {"enabled": config.auth.internal.enabled},
+            },
+            "system": {
+                # "type": config.system.type,
+                "version": get_version(),
+                "branch": get_branch(),
+                "commit": get_commit(),
+            },
+            "ui": {
+                "apps": apps,
+                "cors_origins": config.ui.cors_origins,
+            },
+        },
+        "c12nDef": classification_definition,
+    }

clue/services/fetcher_service.py

@@ -0,0 +1,203 @@
+from typing import Any, Optional
+from urllib.parse import urljoin
+
+import elasticapm
+import requests
+from flask import request
+from pydantic import TypeAdapter, ValidationError
+from requests import JSONDecodeError, exceptions
+
+from clue.common.exceptions import (
+    AuthenticationException,
+    ClueException,
+    ClueValueError,
+    NotFoundException,
+)
+from clue.common.logging import get_logger
+from clue.config import CLASSIFICATION, DEBUG, cache, config
+from clue.models.config import ExternalSource
+from clue.models.fetchers import FetcherDefinition, FetcherResult
+from clue.models.selector import Selector
+from clue.services import auth_service
+
+logger = get_logger(__file__)
+
+# Either cache for one second in debug mode, or five minutes in production
+CACHE_TIMEOUT: int = 1 if DEBUG else 5 * 60
+
+
+@cache.memoize(timeout=1 if DEBUG else 5 * 60, args_to_ignore=["access_token"])  # Cached for 5 minutes
+def get_supported_fetchers(
+    source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
+) -> dict[str, FetcherDefinition]:
+    """Gets all supported fetchers for a source
+
+    Args:
+        source_url (str): The URL of the source
+        access_token (Optional[str], optional): The access token to use, if necessary. Defaults to None.
+
+    Returns:
+        dict[str, FetcherDefinition]: A dict of each ids mapped to fetcher metadata
+    """
+    logger.info("Requesting fetchers for source %s", source.name)
+
+    url = urljoin(source.url, "fetchers/")
+
+    obo_access_token = None
+    if access_token:
+        obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
+
+        if error:
+            logger.error("%s: %s", source.name, error)
+            return {}
+
+    headers = {"Accept": "application/json"}
+    if obo_access_token or access_token:
+        headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
+
+    with elasticapm.capture_span(f"GET {url}", span_type="http"):
+        try:
+            rsp = requests.get(url, headers=headers, timeout=5.0)
+            result = rsp.json()
+
+            if not rsp.ok:
+                err = result["api_error_message"]
+                logger.error(f"Error from upstream server: {rsp.status_code=}, {err=}")
+
+            return TypeAdapter(dict[str, FetcherDefinition]).validate_python(result["api_response"])
+        except exceptions.ConnectionError:
+            # any errors are logged and no result is saved to local cache to enable retry on next query
+            logger.exception("Unable to connect: %s", url)
+            return {}
+        except (requests.exceptions.JSONDecodeError, KeyError):
+            logger.exception("External API did not return expected format:")
+            return {}
+        except ValidationError:
+            logger.exception("ValidationError in response from %s:", source.url)
+            return {}
+
+
+def all_supported_fetchers(user: dict[str, Any], access_token: Optional[str] = None) -> dict[str, FetcherDefinition]:
+    """Gets all supported fetchers for all sources
+
+    Args:
+        access_token (Optional[str], optional): The access token to use, if necessary. Defaults to None.
+
+    Returns:
+        dict[str, FetcherDefinition]: A dict of all fetchers and their matching schema
+    """
+    all_fetchers: dict[str, FetcherDefinition] = {}
+
+    for source in config.api.external_sources:
+        supported_fetchers = get_supported_fetchers(source, user, access_token=access_token)
+        total_fetchers = 0
+        for key, action in supported_fetchers.items():
+            total_fetchers += 1
+            all_fetchers[f"{source.name}.{key}"] = action
+        logger.debug("Plugin %s exposes %s fetcher(s)", source.name, total_fetchers)
+
+    return all_fetchers
+
+
+def get_plugins_supported_fetchers(user: dict[str, Any]) -> dict[str, FetcherDefinition]:
+    """Return the supported fetchers of each external service, filtered to what the user has access to."""
+    available_fetchers: dict[str, FetcherDefinition] = {}
+
+    access_token = request.headers.get("Authorization", type=str)
+    if access_token:
+        access_token = access_token.split(" ")[1]
+
+    all_fetchers = all_supported_fetchers(
+        user,
+        access_token=access_token,
+    )
+
+    logger.info("Retrieving fetchers for classification %s", user["classification"])
+
+    for fetcher_id, fetcher in all_fetchers.items():
+        # Validate if the user is allow to even see the source
+        if user and not CLASSIFICATION.is_accessible(user["classification"], fetcher.classification):
+            logger.info(
+                "Not including fetchers from source %s at classification %s", fetcher.id, user["classification"]
+            )
+            continue
+
+        # user can view source, now filter types user cannot see
+        available_fetchers[fetcher_id] = fetcher
+
+    logger.info("%s fetchers are available for user %s", len(available_fetchers), user["uname"])
+
+    return available_fetchers
+
+
+def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> FetcherResult:
+    """Executes a specified fetcher.
+
+    Args:
+        plugin_id (str): The ID of the plugin.
+        fetcher_id (str): The ID of the action to run.
+        user (dict[str, Any]): The user dict of the user running the action.
+
+    Raises:
+        NotFoundException: Raised whenever the plugin or the action doesn't exist.
+        ClueException: Raised whenever an error is returned by the plugin endpoint.
+
+    Returns:
+        ActionResult: The result of the action.
+    """
+    plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
+
+    if not plugin:
+        raise NotFoundException(f"Plugin {plugin_id} does not exist.")
+
+    access_token = request.headers.get("Authorization", type=str)
+    if access_token:
+        access_token = access_token.split(" ")[1]
+
+    obo_access_token = None
+    if access_token:
+        obo_access_token, error = auth_service.check_obo(plugin, access_token, user["uname"])
+
+        if error:
+            logger.error("%s: %s", plugin.name, error)
+            raise AuthenticationException("Invalid token provided for this enrichment.")
+
+    headers = {"Accept": "application/json"}
+    if obo_access_token or access_token:
+        headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
+
+    if request.content_type == "application/json":
+        parameters = request.json
+    else:
+        # TODO: Pass parameters via urlencode?
+        parameters = {}
+
+    try:
+        Selector.model_validate(parameters)
+
+        response = requests.post(
+            urljoin(plugin.url, f"fetchers/{fetcher_id}"),
+            json=parameters,
+            headers=headers,
+            timeout=request.args.get("max_timeout", 60.0, type=float),
+        )
+
+        result = response.json()
+
+        if not response.ok:
+            raise ClueException(
+                result["api_error_message"] or result["api_response"].get("error", ""), status_code=response.status_code
+            )
+
+        return FetcherResult.model_validate(result["api_response"], context={"is_response": True})
+    except ValidationError as err:
+        logger.exception("Invalid Request Body:")
+        raise ClueValueError(
+            "Validation error encountered on request body. Ensure your request body is properly formatted.",
+            status_code=400,
+        ) from err
+    except (JSONDecodeError, exceptions.ConnectionError) as err:
+        logger.exception(f"Something went wrong when running fetcher from plugin '{plugin_id}'")
+        raise ClueException(
+            f"Something went wrong when running fetcher from plugin '{plugin_id}': {err.__class__.__name__}."
+        ) from err