howler-api 3.0.0.dev374__py3-none-any.whl

howler/services/auth_service.py

@@ -0,0 +1,323 @@
+import base64
+import hashlib
+from datetime import datetime
+from typing import Optional, Union
+
+import elasticapm
+from flask import request
+
+import howler.services.jwt_service as jwt_service
+import howler.services.user_service as user_service
+from howler.common.exceptions import (
+    AccessDeniedException,
+    AuthenticationException,
+    HowlerException,
+    InvalidDataException,
+)
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.config import config, redis
+from howler.odm.models.user import User
+from howler.remote.datatypes.queues.named import NamedQueue
+from howler.remote.datatypes.set import ExpiringSet
+from howler.security.utils import generate_random_secret, verify_password
+
+logger = get_logger(__file__)
+
+nonpersistent_config: dict[str, Union[str, int]] = {
+    "host": config.core.redis.nonpersistent.host,
+    "port": config.core.redis.nonpersistent.port,
+    "ttl": config.auth.internal.failure_ttl,
+}
+
+
+def _get_token_store(user: str) -> ExpiringSet:
+    """Get an expiring redis set in which to add a token
+
+    Args:
+        user (str): The user the token corresponds to
+
+    Returns:
+        ExpiringSet: The set in which we'll store the token
+    """
+    return ExpiringSet(f"token_{user}", host=redis, ttl=60 * 60)  # 1 Hour expiry
+
+
+def _get_priv_store(user: str, token: str) -> ExpiringSet:
+    """Get an expiring redis set in which to add the privileges
+
+    Args:
+        user (str): The user the token corresponds to
+        token (str): The token the privileges correspond to
+
+    Returns:
+        ExpiringSet: The set in which we'll store the privileges
+    """
+    return ExpiringSet(
+        # For security reasons, we won't save the whole token in redis. Just in case :)
+        f"token_priv_{user}_{token[:10]}",
+        host=redis,
+        # 1 Hour expiry
+        ttl=60 * 60,
+    )
+
+
+def create_token(user: str, priv: list[str]) -> str:
+    """Generate a new token associated with the given user with the given privileges
+
+    Args:
+        user (str): The user to create the token as
+        priv (list[str]): The privileges to give the token
+
+    Returns:
+        str: The new token
+    """
+    token = hashlib.sha256(str(generate_random_secret()).encode("utf-8", errors="replace")).hexdigest()
+
+    _get_token_store(user).add(token)
+    priv_store = _get_priv_store(user, token)
+    priv_store.pop_all()
+    priv_store.add(",".join(priv))
+
+    return token
+
+
+def check_token(user: str, token: str) -> Optional[list[str]]:
+    """Check if a token exists, and return its list of privileges
+
+    Args:
+        user (str): The user corresponding to the token to check
+        token (str): The token
+
+    Returns:
+        Optional[list[str]]: The list of privileges associated with the token
+    """
+    if _get_token_store(user).exist(token):
+        members = _get_priv_store(user, token).members()
+        if len(members) > 0:
+            priv_str = members[0]
+            return priv_str.split(",")
+
+    return None
+
+
+def validate_token(username: str, token: str) -> Optional[list[str]]:
+    """This function identifies the user via the internal token functionality
+
+    Args:
+        username (str): The username corresponding to the provided token
+        token (str): The token generated by our API to check for
+
+    Raises:
+        AuthenticationException: Invalid token
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    if token:
+        priv = check_token(username, token)
+        if priv:
+            return priv
+
+        raise AuthenticationException("Invalid token")
+
+    return None
+
+
+@elasticapm.capture_span(span_type="authentication")
+def bearer_auth(
+    data: str, skip_jwt: bool = False, skip_internal: bool = False
+) -> tuple[Optional[User], Optional[list[str]]]:
+    """This function handles Bearer type Authorization headers.
+
+    Args:
+        data (str): The corresponding data in the Authorization header.
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    if "." in data:
+        if not skip_jwt:
+            try:
+                jwt_data = jwt_service.decode(data, validate_audience=True)
+            except HowlerException as e:
+                raise AuthenticationException(
+                    "Something went wrong when decoding your key. Please reauthenticate.",
+                    cause=e,
+                )
+
+            cur_user = user_service.parse_user_data(jwt_data, jwt_service.get_provider(data))
+
+            if cur_user:
+                return cur_user, ["R", "W", "E"]
+
+            return None, None
+        else:
+            raise InvalidDataException("Not a valid authentication type for this endpoint.")
+    else:
+        if not skip_internal:
+            [username, token] = data.split(":", maxsplit=1)
+
+            privs = validate_token(username, token)
+
+            if privs is not None:
+                return datastore().user.get(username), privs
+
+            return None, None
+        else:
+            raise InvalidDataException("Not a valid authentication type for this endpoint.")
+
+
+@elasticapm.capture_span(span_type="authentication")
+def validate_apikey(
+    username: str, apikey: str, impersonator: Optional[User] = None
+) -> tuple[Optional[User], Optional[list[str]]]:
+    """This function identifies the user via the internal API key functionality.
+
+    Args:
+        username (str): The username corresponding to the provided api key
+        apikey (str): The apikey used to authenticate as the user
+        impersonator (Optional[str]): The user who wants to impersonate as the provided username. Defaults to None.
+
+    Raises:
+        AccessDeniedException: Api Key authentication was disabled, or the api was not valid for impersonation,
+                               or it was an impersonation api key incorrectly provided in the Authorization header.
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    if config.auth.allow_apikeys and apikey:
+        user_data: User = datastore().user.get_if_exists(username)
+        if user_data:
+            try:
+                # Get the name and secret data of the api key we are validating
+                name, apikey_password = apikey.split(":", 1)
+                key = user_data.apikeys.get(name, None)
+
+                # Does the key actually exist?
+                if not key:
+                    raise AuthenticationException("API Key does not exist")
+
+                if key.expiry_date is not None:
+                    if key.expiry_date.replace(tzinfo=None) < datetime.utcnow():
+                        raise AuthenticationException("Key is expired")
+
+                # Handle impersonation. Basically, make sure that either:
+                # a) someone is trying to impersonate as this user, and the apikey can be used for that, AND the
+                #    impersonator is on the list of people allowed to use it
+                # b) The user is not being impersonated, and the api key isn't specifically meant for impersonation
+                if impersonator and ("I" not in key.acl or impersonator["uname"] not in key.agents):
+                    raise AccessDeniedException("Not a valid impersonation api key")
+                elif not impersonator and "I" in key.acl:
+                    raise AccessDeniedException(
+                        "Cannot use impersonation key in normal Authorization Header! "
+                        + "Provide your credentials and supply it in the X-Impersonating header instead."
+                    )
+
+                # If the key can be used for whichever purpose, actually validate the secret data
+                if verify_password(apikey_password, key.password):
+                    return user_data, key.acl
+            except ValueError:
+                pass
+
+        return None, None
+    else:
+        raise AccessDeniedException("API Key authentication disabled")
+
+
+def validate_userpass(username: str, password: str) -> tuple[Optional[User], Optional[list[str]]]:
+    """This function identifies the user via the user/pass functionality
+
+    Args:
+        username (str): The username corresponding to the provided password
+        password (str): The password used to authenticate as the user
+
+    Raises:
+        AccessDeniedException: Username/Password authentication is currently disabled
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    if config.auth.internal.enabled and username and password:
+        user = datastore().user.get(username)
+        if user:
+            if verify_password(password, user.password):
+                return user, ["R", "W", "E"]
+
+        return None, None
+    else:
+        raise AccessDeniedException("Username/Password authentication disabled")
+
+
+def decode_b64(b64_str: str) -> str:
+    """Decode a base64 string into plain text.
+
+    Args:
+        b64_str (str): The base64 string
+
+    Raises:
+        InvalidDataException: The data was not base64.
+
+    Returns:
+        str: A plain text representation of the data.
+    """
+    try:
+        return base64.b64decode(b64_str).decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise InvalidDataException("Basic authentication data must be base64 encoded") from e
+
+
+@elasticapm.capture_span(span_type="authentication")
+def basic_auth(
+    data: str, is_base64: bool = True, skip_apikey: bool = False, skip_password: bool = False
+) -> tuple[Optional[User], Optional[list[str]]]:
+    """This function handles Basic type Authorization headers.
+
+    Args:
+        data (str): The corresponding data in the Authorization header.
+        is_base64 (bool, optional): Whether the provided data is base64 encoded. Defaults to True.
+        skip_apikey (bool, optional): Whether to skip apikey validation. Defaults to False.
+        skip_password (bool, optional): Whether to skip password validation. Defaults to False.
+
+    Raises:
+        AuthenticationException: The login information is invalid, or the maximum password retry for the account
+                                 has been reached.
+
+    Returns:
+        tuple[Optional[User], Optional[list[str]]]: The user odm object and privileges, if validated
+    """
+    key_pair = decode_b64(data) if is_base64 else data
+
+    [username, data] = key_pair.split(":", maxsplit=1)
+
+    validated_user = None
+    if not skip_apikey:
+        validated_user, priv = validate_apikey(username, data)
+
+    # Bruteforce protection
+    auth_fail_queue: NamedQueue = NamedQueue(f"ui-failed-{username}", **nonpersistent_config)  # type: ignore
+    if auth_fail_queue.length() >= config.auth.internal.max_failures:
+        # Failed 'max_failures' times, stop trying... This will timeout in 'failure_ttl' seconds
+        raise AuthenticationException(
+            "Maximum password retry of {retry} was reached. "
+            "This account is locked for the next {ttl} "
+            "seconds...".format(
+                retry=config.auth.internal.max_failures,
+                ttl=config.auth.internal.failure_ttl,
+            )
+        )
+
+    if not validated_user and not skip_password:
+        validated_user, priv = validate_userpass(username, data)
+
+    if not validated_user:
+        auth_fail_queue.push(
+            {
+                "remote_addr": request.remote_addr,
+                "host": request.host,
+                "full_path": request.full_path,
+            }
+        )
+        raise AuthenticationException("Invalid login information")
+
+    return validated_user, priv

howler/services/config_service.py

@@ -0,0 +1,128 @@
+from datetime import datetime
+from math import ceil
+from typing import Optional
+
+from flask import request
+
+import howler.services.hit_service as hit_service
+from howler.common.exceptions import ForbiddenException, HowlerException
+from howler.common.loader import get_lookups
+from howler.common.logging import get_logger
+from howler.config import CLASSIFICATION, config, get_branch, get_commit, get_version
+from howler.helper.discover import get_apps_list
+from howler.helper.search import list_all_fields
+from howler.odm.models.howler_data import Assessment, Escalation, HitStatus, Scrutiny
+from howler.odm.models.user import User
+from howler.plugins import get_plugins
+from howler.services import jwt_service
+from howler.utils.str_utils import default_string_value
+
+classification_definition = CLASSIFICATION.get_parsed_classification_definition()
+
+lookups = get_lookups()
+
+logger = get_logger()
+
+
+def _get_apikey_max_duration():
+    "Configure the maximum duration of a created API key"
+    amount, unit = (
+        config.auth.max_apikey_duration_amount,
+        config.auth.max_apikey_duration_unit,
+    )
+
+    if not config.auth.oauth.strict_apikeys:
+        return amount, unit
+
+    auth_header: Optional[str] = request.headers.get("Authorization", None)
+
+    if not auth_header:
+        return amount, unit
+
+    if not auth_header.startswith("Bearer") or "." not in auth_header:
+        return amount, unit
+
+    oauth_token = auth_header.split(" ")[1]
+    try:
+        data = jwt_service.decode(
+            oauth_token,
+            validate_audience=False,
+            options={"verify_signature": False},
+        )
+        amount, unit = (
+            ceil((datetime.fromtimestamp(data["exp"]) - datetime.now()).total_seconds()),
+            "seconds",
+        )
+    except ForbiddenException:
+        logger.warning("Access token is expired.")
+    except HowlerException:
+        logger.exception("Error occurred when decoding access token.")
+    finally:
+        return amount, unit
+
+
+def get_configuration(user: User | None, **kwargs):
+    """Get system configration data for the Howler API
+
+    Args:
+        user (User): The user making the request
+    """
+    apps = get_apps_list(discovery_url=kwargs.get("discovery_url", None))
+
+    amount, unit = _get_apikey_max_duration()
+
+    plugin_features: dict[str, bool] = {}
+
+    for plugin in get_plugins():
+        try:
+            plugin_features = {**plugin_features, **plugin.features}
+        except (ImportError, AttributeError):
+            pass
+
+    return {
+        "lookups": {
+            "howler.status": HitStatus.list(),
+            "howler.scrutiny": Scrutiny.list(),
+            "howler.escalation": Escalation.list(),
+            "howler.assessment": Assessment.list(),
+            "transitions": {status: hit_service.get_transitions(status) for status in HitStatus.list()},
+            **lookups,
+        },
+        "configuration": {
+            "auth": {
+                "allow_apikeys": config.auth.allow_apikeys,
+                "allow_extended_apikeys": config.auth.allow_extended_apikeys,
+                "max_apikey_duration_amount": amount,
+                "max_apikey_duration_unit": unit,
+                "oauth_providers": [
+                    name
+                    for name, p in config.auth.oauth.providers.items()
+                    if default_string_value(p.client_secret, env_name=f"{name.upper()}_CLIENT_SECRET")
+                ],
+                "internal": {"enabled": config.auth.internal.enabled},
+            },
+            "system": {
+                "type": config.system.type,
+                "version": get_version(),
+                "branch": get_branch(),
+                "commit": get_commit(),
+                "retention": {
+                    "enabled": config.system.retention.enabled,
+                    "limit_unit": config.system.retention.limit_unit,
+                    "limit_amount": config.system.retention.limit_amount,
+                },
+            },
+            "ui": {
+                "apps": apps,
+            },
+            "mapping": config.mapping,
+            "features": {
+                "clue": config.core.clue.enabled,
+                "notebook": config.core.notebook.enabled,
+                **plugin_features,
+            },
+            "clue": {"status_checks": config.core.clue.status_checks},
+        },
+        "c12nDef": classification_definition,
+        "indexes": list_all_fields("admin" in user["type"] if user is not None else False),
+    }

howler/services/dossier_service.py

@@ -0,0 +1,252 @@
+"""Dossier service module for managing security investigation dossiers.
+
+This module provides functionality for creating, updating, retrieving, and managing
+dossiers - collections of security alerts and investigation data organized by analysts.
+Dossiers can be personal (private to the creator) or global (shared with the team).
+"""
+
+from typing import Any, Optional, cast
+
+from mergedeep.mergedeep import merge
+
+from howler.common.exceptions import ForbiddenException, HowlerException, InvalidDataException, NotFoundException
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.datastore.exceptions import SearchException
+from howler.odm.models.dossier import Dossier
+from howler.odm.models.user import User
+from howler.services import lucene_service
+
+logger = get_logger(__file__)
+
+# Define which fields are allowed to be updated in a dossier, preventing unauthorized modification of sensitive fields
+PERMITTED_KEYS = {
+    "title",
+    "query",
+    "leads",
+    "pivots",
+    "type",
+    "owner",
+}
+
+
+def exists(dossier_id: str) -> bool:
+    """Check if a dossier exists in the datastore.
+
+    Args:
+        dossier_id: Unique identifier for the dossier
+
+    Returns:
+        True if the dossier exists, False otherwise
+    """
+    return datastore().dossier.exists(dossier_id)
+
+
+def get_dossier(
+    id: str,
+    as_odm: bool = False,
+    version: bool = False,
+) -> Dossier:
+    """Retrieve a dossier from the datastore.
+
+    Args:
+        id: Unique identifier for the dossier
+        as_odm: Whether to return as ODM object (True) or dictionary (False)
+        version: Whether to include version information in the response
+
+    Returns:
+        Dossier object or dictionary containing dossier data
+
+    Raises:
+        NotFoundException: If the dossier doesn't exist
+    """
+    return datastore().dossier.get_if_exists(key=id, as_obj=as_odm, version=version)
+
+
+def create_dossier(dossier_data: Optional[Any], username: str) -> Dossier:  # noqa: C901
+    """Create a new dossier in the datastore.
+
+    This function validates the input data, ensures the query is valid by testing it
+    against the hit collection, and creates a new dossier with the specified parameters.
+
+    Args:
+        dossier_data: Dictionary containing dossier configuration data
+        username: Username of the user creating the dossier
+
+    Returns:
+        Newly created Dossier object
+
+    Raises:
+        InvalidDataException: If data format is invalid, required fields are missing,
+                            or the query is invalid
+        HowlerException: If there's an error during dossier creation
+    """
+    # Validate input data format
+    if not isinstance(dossier_data, dict):
+        raise InvalidDataException("Invalid data format")
+
+    # Validate required fields for dossier creation
+    if "title" not in dossier_data:
+        raise InvalidDataException("You must specify a title when creating a dossier.")
+
+    if "query" not in dossier_data:
+        raise InvalidDataException("You must specify a query when creating a dossier.")
+
+    if "type" not in dossier_data:
+        raise InvalidDataException("You must specify a type when creating a dossier.")
+
+    storage = datastore()
+
+    try:
+        # Validate the Lucene query by attempting to search with it
+        # This ensures the query syntax is correct before saving the dossier
+        if query := dossier_data.get("query", None):
+            storage.hit.search(query)
+
+        if "owner" not in dossier_data:
+            dossier_data["owner"] = username
+
+        dossier = Dossier(dossier_data)
+
+        # Validate pivot configurations to ensure no duplicate mapping keys
+        for pivot in dossier.pivots:
+            if len(pivot.mappings) != len(set(mapping.key for mapping in pivot.mappings)):
+                raise InvalidDataException("One of your pivots has duplicate keys set.")
+
+        # Ensure the owner is set to the current user (security measure)
+        dossier.owner = username
+
+        # Save the dossier to the datastore
+        storage.dossier.save(dossier.dossier_id, dossier)
+
+        # Commit the transaction to persist changes
+        storage.dossier.commit()
+
+        return dossier
+    except SearchException:
+        # Handle invalid Lucene query syntax
+        raise InvalidDataException("You must use a valid query when creating a dossier.")
+    except HowlerException as e:
+        # Handle other application-specific errors
+        raise InvalidDataException(str(e))
+
+
+def update_dossier(dossier_id: str, dossier_data: dict[str, Any], user: User) -> Dossier:  # noqa: C901
+    """Update one or more properties of a dossier in the database.
+
+    This function enforces access control rules and validates data before updating.
+    Personal dossiers can only be updated by their owners or admins.
+    Global dossiers can only be updated by their owners or admins.
+
+    Args:
+        dossier_id: Unique identifier of the dossier to update
+        dossier_data: Dictionary containing fields to update
+        user: User object representing the requesting user
+
+    Returns:
+        Updated Dossier object
+
+    Raises:
+        NotFoundException: If the dossier doesn't exist
+        InvalidDataException: If invalid fields are provided or data is malformed
+        ForbiddenException: If user lacks permission to update the dossier
+    """
+    # Verify the dossier exists before attempting to update
+    if not exists(dossier_id):
+        raise NotFoundException(f"Dossier with id '{dossier_id}' does not exist.")
+
+    # Validate that only permitted fields are being updated
+    # This prevents unauthorized modification of sensitive fields
+    if set(dossier_data.keys()) - PERMITTED_KEYS:
+        raise InvalidDataException(f"Only {', '.join(PERMITTED_KEYS)} can be updated.")
+
+    storage = datastore()
+
+    # Retrieve the existing dossier for access control checks
+    existing_dossier: Dossier = get_dossier(dossier_id, as_odm=True)
+
+    # Enforce access control for personal dossiers
+    # Only the owner or admin users can modify personal dossiers
+    if existing_dossier.type == "personal" and existing_dossier.owner != user.uname and "admin" not in user.type:
+        raise ForbiddenException("You cannot update a personal dossier that is not owned by you.")
+
+    # Enforce access control for global dossiers
+    # Only the owner or admin users can modify global dossiers
+    if existing_dossier.type == "global" and existing_dossier.owner != user.uname and "admin" not in user.type:
+        raise ForbiddenException("Only the owner of a dossier and administrators can edit a global dossier.")
+
+    # Validate pivot configurations if they're being updated
+    # Ensure no duplicate mapping keys exist within any pivot
+    if "pivots" in dossier_data:
+        for pivot in dossier_data["pivots"]:
+            if len(pivot["mappings"]) != len(set(mapping["key"] for mapping in pivot["mappings"])):
+                raise InvalidDataException("One of your pivots has duplicate keys set.")
+
+    try:
+        # Validate the Lucene query if it's being updated
+        if "query" in dossier_data:
+            # Test the query against the hit index to ensure it's valid
+            storage.hit.search(dossier_data["query"])
+
+        # Merge the new data with existing dossier data
+        new_data = Dossier(merge({}, existing_dossier.as_primitives(), dossier_data))
+
+        storage.dossier.save(dossier_id, new_data)
+
+        # Commit the transaction to persist changes
+        storage.dossier.commit()
+
+        return new_data
+    except SearchException:
+        # Handle invalid Lucene query syntax
+        raise InvalidDataException("You must use a valid query when updating a dossier.")
+    except (HowlerException, TypeError) as e:
+        # Log the error for debugging purposes
+        logger.exception("Error when updating dossier.")
+        # Provide a user-friendly error message while preserving the original exception
+        raise InvalidDataException("We were unable to update the dossier.", cause=e) from e
+
+
+def get_matching_dossiers(hit: dict[str, Any], dossiers: Optional[list[dict[str, Any]]] = None):
+    """Get a list of dossiers that match a specific security alert/hit.
+
+    This function evaluates each dossier's query against the provided hit data
+    to determine which dossiers are relevant to the security event.
+
+    Args:
+        hit: Dictionary containing security alert/hit data to match against
+        dossiers: Optional list of dossiers to check. If None, all dossiers
+                 will be retrieved from the datastore
+
+    Returns:
+        List of dossier dictionaries that match the provided hit
+
+    Note:
+        This function uses Lucene query matching to determine relevance.
+        Dossiers with no query are assumed to match all hits.
+    """
+    # Retrieve all dossiers if none provided
+    if dossiers is None:
+        dossiers: list[dict[str, Any]] = datastore().dossier.search(
+            "dossier_id:*",
+            as_obj=False,
+            # TODO: Eventually implement caching here
+            rows=1000,
+        )["items"]
+
+    matching_dossiers: list[dict[str, Any]] = []
+
+    # Evaluate each dossier against the hit data
+    for dossier in cast(list[dict[str, Any]], dossiers):
+        # Dossiers without queries match all hits by default
+        # This allows for catch-all dossiers that collect all security events
+        if "query" not in dossier or dossier["query"] is None:
+            matching_dossiers.append(dossier)
+            continue
+
+        # Use Lucene service to check if the hit matches the dossier's query
+        # This determines if the security event is relevant to this investigation
+        if lucene_service.match(dossier["query"], hit):
+            matching_dossiers.append(dossier)
+
+    return matching_dossiers