howler-api 3.0.0.dev374__py3-none-any.whl

howler/helper/hit.py

@@ -0,0 +1,236 @@
+from typing import Any, Optional, Union
+
+from howler.common.exceptions import InvalidDataException
+from howler.common.logging import get_logger
+from howler.datastore.operations import OdmHelper, OdmUpdateOperation
+from howler.helper.workflow import Transition
+from howler.odm.models.hit import Hit
+from howler.odm.models.howler_data import (
+    Assessment,
+    AssessmentEscalationMap,
+    Escalation,
+    HitStatus,
+    HitStatusTransition,
+    Vote,
+)
+from howler.odm.models.user import User
+
+odm_helper = OdmHelper(Hit)
+
+logger = get_logger(__name__)
+
+
+def assess_hit(
+    assessment: Optional[str] = None,
+    rationale: Optional[str] = None,
+    hit: Optional[Union[dict[str, Any], Hit]] = None,
+    **kwargs,
+) -> list[OdmUpdateOperation]:
+    """Update the assessment and esclation of a hit
+
+    Args:
+        assessment (Optional[str], optional): The assessment to set the hit to. Defaults to None.
+        hit (Optional[dict[str, Any]], optional): The hit to update. Defaults to None.
+
+    Raises:
+        InvalidDataException: An invalid assessment was provided
+
+    Returns:
+        list[OdmUpdateOperation]: A list of the opperations to run on the hit
+    """
+    escalation: Optional[str] = None
+    if not assessment:
+        # In case the assessment is set to empty string
+        assessment = None
+    else:
+        if assessment not in Assessment:
+            assessment_list = ", ".join(Assessment)
+            raise InvalidDataException(f"Must set assessment to one of {assessment_list}.")
+
+        escalation = AssessmentEscalationMap[assessment]
+
+    if assessment is None and rationale:
+        rationale = None
+
+    logger.debug(
+        "Updating assessment of %s to %s",
+        hit["howler"]["id"] if hit else "unknown",
+        assessment,
+    )
+    logger.debug(
+        "Updating escalation of %s to %s",
+        hit["howler"]["id"] if hit else "unknown",
+        escalation,
+    )
+
+    return [
+        odm_helper.update("howler.assessment", assessment),
+        odm_helper.update("howler.escalation", escalation),
+        odm_helper.update("howler.rationale", rationale, silent=True),
+    ]
+
+
+def unassign_hit(
+    hit: dict[str, Any],
+    user: Optional[User] = None,
+    **kwargs,
+) -> list[OdmUpdateOperation]:
+    """Remove the assignment of a hit
+
+    Args:
+        user (Optional[User], optional): The user unassigning the hit. Defaults to None.
+        hit (Optional[dict[str, Any]], optional): The hit to unassign the user from. Defaults to None.
+
+    Raises:
+        InvalidDataException: The user unassigning the hit doesn't have the hit assigned to them
+
+    Returns:
+        list[OdmUpdateOperation]: A list of the operations necessary to update the hit
+    """
+    if user and hit["howler"]["assignment"] == user.get("uname", user.get("username", None)):
+        return [odm_helper.update("howler.assignment", "unassigned")]
+
+    raise InvalidDataException("Cannot release hit that isn't assigned to you.")
+
+
+def assign_hit(
+    transition: Transition,
+    user: Optional[User] = None,
+    assignee: Optional[str] = None,
+    hit: Optional[dict[str, Any]] = None,
+    **kwargs,
+) -> list[OdmUpdateOperation]:
+    """Assign a hit to a user
+
+    Args:
+        transition (Transition): The type of transition being used to assign the hit
+        user (Optional[User], optional): The user assigning the hit. Defaults to None.
+        assignee (Optional[str], optional): The user to assign the hit to. Defaults to None.
+        hit (Optional[dict[str, Any]], optional): The hit we are assigning. Defaults to None.
+
+    Raises:
+        InvalidDataException: Incorrect parameters were provided
+
+    Returns:
+        list[OdmUpdateOperation]: A list of operations to update the hit assignment
+    """
+    if transition["transition"] == HitStatusTransition.ASSIGN_TO_OTHER:
+        if not assignee:
+            raise InvalidDataException("Must specify an assignee when assigning to another user.")
+
+        if hit and hit["howler"]["assignment"] == assignee:
+            raise InvalidDataException("Must specify an assignee that is different from the current assigned user.")
+
+    if not user and not assignee:
+        raise InvalidDataException("Could not assign Hit to user a no 'user_id' was provided")
+
+    return [
+        odm_helper.update(
+            "howler.assignment",
+            assignee or user.get("uname", user.get("username", None)) if user else None,
+        )
+    ]
+
+
+def check_ownership(
+    hit: dict[str, Any],
+    user: Optional[dict[str, Any]] = None,
+    **kwargs,
+) -> list[OdmUpdateOperation]:
+    """Check the ownership of a hit, and throw an exception if it doesnt match
+
+    Args:
+        hit (dict[str, Any]): The hit to check
+        user (Optional[dict[str, Any]], optional): The user to check for ownership of. Defaults to None.
+
+    Raises:
+        InvalidDataException: Raised when the hit assignee doesn't match the user
+
+    Returns:
+        list[OdmUpdateOperation]: An empty list
+    """
+    if user and hit["howler"]["assignment"] != user.get("uname", user.get("username", None)):
+        raise InvalidDataException("Cannot use this transition when the hit is not assigned to you.")
+
+    return []
+
+
+def promote_hit(**kwargs) -> list[OdmUpdateOperation]:
+    """Promote a hit to an alert
+
+    Returns:
+        list[OdmUpdateOperation]: The update to run to promote
+    """
+    return [odm_helper.update("howler.escalation", kwargs.get("escalation", Escalation.ALERT))]
+
+
+def demote_hit(**kwargs) -> list[OdmUpdateOperation]:
+    """Demote an alert to a hit
+
+    Returns:
+        list[OdmUpdateOperation]: The update to run to demote
+    """
+    return [odm_helper.update("howler.escalation", kwargs.get("escalation", Escalation.HIT))]
+
+
+def vote_hit(
+    hit: dict[str, Any],
+    vote: str,
+    email: str,
+    user: Optional[dict[str, Any]] = None,
+    **kwargs,
+) -> list[OdmUpdateOperation]:
+    """Add a vote to the given hit
+
+    Args:
+        hit (dict[str, Any]): The hit to add the vote to
+        vote (str): The type of vote to add
+        email (str): The email of the user voting
+        user (Optional[dict[str, Any]], optional): The user voting. Defaults to None.
+
+    Raises:
+        InvalidDataException: Invalid data was provided
+
+    Returns:
+        list[OdmUpdateOperation]: A list of operations to update the hit depending on the vote
+    """
+    if not email:
+        raise InvalidDataException("Could not vote on Hit as no email was provided")
+
+    if vote not in Vote or vote == "" or vote is None:
+        raise InvalidDataException(f"vote is not optional. Provide a value from: {', '.join(Vote)}")
+
+    actions = []
+
+    # Check to see if there is an existing vote from this user
+    old_vote = (
+        "benign"
+        if email in hit["howler"]["votes"]["benign"]
+        else (
+            "obscure"
+            if email in hit["howler"]["votes"]["obscure"]
+            else "malicious"
+            if email in hit["howler"]["votes"]["malicious"]
+            else None
+        )
+    )
+
+    if old_vote:
+        logger.debug("removing old vote of %s from %s", old_vote, id)
+        actions.append(odm_helper.list_remove(f"howler.votes.{old_vote}", email))
+
+    if not old_vote or old_vote != vote:
+        logger.debug("Adding vote of %s to %s", vote, id)
+        actions.append(odm_helper.list_add(f"howler.votes.{vote}", email, if_missing=True))
+
+    if user and hit["howler"]["assignment"] == user.get("uname", user.get("username", None)):
+        if hit["howler"]["status"] in [
+            HitStatus.IN_PROGRESS,
+            HitStatus.OPEN,
+        ]:
+            actions.append(odm_helper.update("howler.assignment", "unassigned"))
+            actions.append(odm_helper.update("howler.status", HitStatus.OPEN))
+        else:
+            raise InvalidDataException("Cannot vote on hit you are assigned to.")
+
+    return actions

howler/helper/oauth.py

@@ -0,0 +1,247 @@
+import base64
+import hashlib
+import re
+from typing import Any, Optional
+
+import elasticapm
+import requests
+
+from howler.common.exceptions import HowlerException, HowlerValueError
+from howler.common.loader import USER_TYPES
+from howler.common.logging import get_logger
+from howler.common.random_user import random_user
+from howler.config import CLASSIFICATION as CLASSIFICATION_ENGINE
+from howler.config import config
+from howler.helper.azure import azure_obo
+from howler.odm.models.config import OAuthProvider
+from howler.services import jwt_service
+
+VALID_CHARS = [str(x) for x in range(10)] + [chr(x + 65) for x in range(26)] + [chr(x + 97) for x in range(26)] + ["-"]
+
+logger = get_logger(__file__)
+
+
+def reorder_name(name: Optional[str]) -> Optional[str]:
+    """Reorder the name from Doe, John to John Doe"""
+    if name is None:
+        return name
+
+    return " ".join(name.split(", ", 1)[::-1])
+
+
+@elasticapm.capture_span(span_type="authentication")
+def parse_profile(profile: dict[str, Any], provider_config: OAuthProvider) -> dict[str, Any]:  # noqa: C901
+    """Parse a raw profile dict into a useful user data dict"""
+    # Find email address and normalize it for further processing
+    email_adr = profile.get(
+        "email",
+        profile.get("emails", profile.get("preferred_username", profile.get("upn", None))),
+    )
+
+    if isinstance(email_adr, list):
+        email_adr = email_adr[0]
+
+    if email_adr:
+        email_adr = email_adr.lower()
+        if "@" not in email_adr:
+            email_adr = None
+
+    # Find the name of the user
+    name = reorder_name(profile.get("name", profile.get("displayName", None)))
+
+    # Generate a username
+    if provider_config.uid_randomize:
+        # Use randomizer
+        uname = random_user(
+            digits=provider_config.uid_randomize_digits,
+            delimiter=provider_config.uid_randomize_delimiter,
+        )
+    else:
+        # Try and find the username
+        uname = profile.get("uname", profile.get("preferred_username", email_adr))
+
+        # Did we default to email?
+        if email_adr is not None and uname is not None and uname.lower() == email_adr.lower():
+            # 1. Use provided regex matcher
+            if provider_config.uid_regex:
+                match = re.match(provider_config.uid_regex, uname)
+                if match:
+                    if provider_config.uid_format:
+                        uname = provider_config.uid_format.format(*[x or "" for x in match.groups()]).lower()
+                    else:
+                        uname = "".join([x for x in match.groups() if x]).lower()
+
+            # 2. Parse name and domain from email if regex failed or missing
+            if uname is not None and uname == email_adr:
+                e_name, e_dom = uname.split("@", 1)
+                uname = f"{e_name}-{e_dom.split('.')[0]}"
+
+        # 3. Use name as username if there are no username found yet
+        if uname is None and name is not None:
+            uname = name.replace(" ", "-")
+
+        # Cleanup username
+        if uname:
+            uname = "".join([c for c in uname if c in VALID_CHARS])
+
+    # Get avatar from gravatar
+    if config.auth.oauth.gravatar_enabled and email_adr:
+        email_hash = hashlib.md5(email_adr.encode("utf-8")).hexdigest()  # noqa: S324
+        alternate = f"https://www.gravatar.com/avatar/{email_hash}?s=256&d=404&r=pg"
+    else:
+        alternate = None
+
+    # Compute access, roles and classification using auto_properties
+    access = True
+    roles = ["user"]
+    classification = CLASSIFICATION_ENGINE.UNRESTRICTED
+    if provider_config.auto_properties:
+        for auto_prop in provider_config.auto_properties:
+            if auto_prop.type == "access":
+                # Set default access value for access pattern
+                access = auto_prop.value != "True"
+
+            # Get values for field
+            field_data = profile.get(auto_prop.field, None)
+            if not isinstance(field_data, list):
+                field_data = [field_data]
+
+            # Analyse field values
+            for value in field_data:
+                # If there is no value, no need to do any tests
+                if value is None:
+                    continue
+
+                # Check access
+                if auto_prop.type == "access":
+                    if re.match(auto_prop.pattern, value) is not None:
+                        access = auto_prop.value == "True"
+                        break
+
+                # Append roles from matching patterns
+                elif auto_prop.type == "role":
+                    if re.match(auto_prop.pattern, value):
+                        roles.append(auto_prop.value)
+                        break
+
+                # Compute classification from matching patterns
+                elif auto_prop.type == "classification":
+                    if re.match(auto_prop.pattern, value):
+                        classification = CLASSIFICATION_ENGINE.build_user_classification(
+                            classification, auto_prop.value
+                        )
+                        break
+
+    # Infer roles from groups
+    if profile.get("groups") and provider_config.role_map:
+        for user_type in USER_TYPES:
+            if (
+                user_type in provider_config.role_map
+                and provider_config.role_map[user_type] in profile.get("groups", [])
+                and user_type not in roles
+            ):
+                roles.append(user_type)
+
+    return dict(
+        access=access,
+        type=roles,
+        classification=classification,
+        uname=uname,
+        name=name,
+        email=email_adr,
+        password="__NO_PASSWORD__",  # noqa: S106
+        avatar=profile.get("picture", provider_config.picture_url or alternate),
+        groups=profile.get("groups", []),
+    )
+
+
+def fetch_avatar(  # noqa: C901
+    url: str, provider: dict[str, Any], oauth_provider: str, access_token: Optional[str] = None
+):
+    """Fetch a user's avatar form the oauth provider"""
+    provider_config = config.auth.oauth.providers[oauth_provider]
+
+    logger.info("Fetching avatar from %s at %s", oauth_provider, url)
+
+    try:
+        # Generic picture url endpoint, i.e. MS Graph
+        if url == provider_config.picture_url:
+            headers = {}
+
+            if oauth_provider == "azure":
+                if not access_token:
+                    raise HowlerValueError("An azure access token is necessary to retrieve the profile picture")  # noqa: TRY301
+
+                token = azure_obo(access_token)
+
+            if token:
+                headers["Authorization"] = f"Bearer {token}"
+
+            resp: Any = requests.get(url, headers=headers, timeout=10)
+
+            if resp.ok and resp.headers.get("content-type") is not None:
+                b64_img = base64.b64encode(resp.content).decode()
+                avatar = f'data:{resp.headers.get("content-type")};base64,{b64_img}'
+                return avatar
+
+        # Url that is protected through OAuth
+        elif provider_config.api_base_url and url.startswith(provider_config.api_base_url):
+            resp = provider.get(url[len(provider_config.api_base_url) :])
+            if resp.ok and resp.headers.get("content-type") is not None:
+                b64_img = base64.b64encode(resp.content).decode()
+                avatar = f'data:{resp.headers.get("content-type")};base64,{b64_img}'
+                return avatar
+
+        # Unprotected url
+        elif url.startswith(("https://", "http://")):
+            resp = requests.get(url, timeout=10)
+            if resp.ok and resp.headers.get("content-type") is not None:
+                b64_img = base64.b64encode(resp.content).decode()
+                avatar = f'data:{resp.headers.get("content-type")};base64,{b64_img}'
+                return avatar
+
+    # Quietly fail, it'll use gravatar instead
+    except Exception as e:
+        logger.warning("Error while retrieving user profile: %s", str(e))
+        return None
+
+
+def fetch_groups(token: str):
+    """Fetch a user's groups form an external endpoint"""
+    oauth_provider = jwt_service.get_provider(token)
+    oauth_provider_config = config.auth.oauth.providers[oauth_provider]
+
+    if oauth_provider_config.groups_url:
+        if oauth_provider == "azure":
+            try:
+                token = azure_obo(token)
+            except HowlerException:
+                logger.exception("Exception on fetching groups data")
+                raise HowlerException("Something went wrong when getting the detailed groups data.")
+
+        headers = {}
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+
+        resp = requests.get(oauth_provider_config.groups_url, headers=headers, timeout=10)
+
+        if resp.ok and resp.headers.get("content-type") is not None:
+            result = resp.json()
+            if oauth_provider_config.groups_key:
+                for part in oauth_provider_config.groups_key.split("."):
+                    result = result[part]
+
+            detailed_group_data = []
+            for group in result:
+                detailed_group_data.append(
+                    {
+                        "id": group.get("id", None),
+                        "name": group.get("name", group.get("displayName", group.get("id", None))),
+                    }
+                )
+
+            return sorted(detailed_group_data, key=lambda g: g.get("name", "").lower())
+
+        raise HowlerException("Something went wrong when getting the detailed groups data.")
+    else:
+        return None

howler/helper/search.py

@@ -0,0 +1,92 @@
+from typing import Any, Callable, Optional, Union
+
+from howler.common.loader import datastore
+from howler.datastore.collection import ESCollection
+from howler.odm.models.user import User
+
+# List of indices where queries are protected with classification access control
+ACCESS_CONTROLLED_INDICES: dict[str, ESCollection] = {}
+
+ADMIN_INDEX_MAP: dict[str, Callable[[], ESCollection]] = {}
+
+ADMIN_INDEX_ORDER_MAP: dict[str, str] = {}
+
+INDEX_MAP: dict[str, Callable[[], ESCollection]] = {
+    "action": lambda: datastore().action,
+    "analytic": lambda: datastore().analytic,
+    "dossier": lambda: datastore().dossier,
+    "hit": lambda: datastore().hit,
+    "overview": lambda: datastore().overview,
+    "template": lambda: datastore().template,
+    "user": lambda: datastore().user,
+    "view": lambda: datastore().view,
+}
+
+INDEX_ORDER_MAP: dict[str, str] = {
+    "action": "name asc",
+    "analytic": "name asc",
+    "dossier": "title asc",
+    "hit": "event.created desc",
+    "overview": "overview_id asc",
+    "template": "template_id asc",
+    "user": "id asc",
+    "view": "title asc",
+}
+
+
+def get_collection(index: str, user: Union[User, dict[str, Any]]) -> Optional[Callable[[], ESCollection]]:
+    """Get the ESCollection for a given index
+
+    Args:
+        index (str): The name of the ESCollection to retrieve
+        user (User): The user retrieving the collection
+
+    Returns:
+        ESCollection: The corresponding ESCollection
+    """
+    return INDEX_MAP.get(index, ADMIN_INDEX_MAP.get(index, None) if "admin" in user["type"] else None)
+
+
+def get_default_sort(index: str, user: Union[User, dict[str, Any]]) -> Optional[str]:
+    """Retrieve the default sorting for a given index
+
+    Args:
+        index (str): The index to get the default sort of
+        user (Union[User, dict[str, Any]]): The user retrieving the collection
+
+    Returns:
+        str: The default sort for the index
+    """
+    return INDEX_ORDER_MAP.get(
+        index,
+        ADMIN_INDEX_ORDER_MAP.get(index, None) if "admin" in user["type"] else None,
+    )
+
+
+def has_access_control(index: str) -> bool:
+    """Check if the given index has access control enabled
+
+    Args:
+        index (str): The index to check
+
+    Returns:
+        bool: Does the index have access control
+    """
+    return index in ACCESS_CONTROLLED_INDICES
+
+
+def list_all_fields(is_admin: bool = False) -> dict[str, dict]:
+    """Generate a list of all fields in each index
+
+    Args:
+        is_admin (bool, optional): Should administrator only indexes be included? Defaults to False.
+
+    Returns:
+        dict[str, dict]: A list of all fields in each index
+    """
+    fields_map = {k: INDEX_MAP[k]().fields(skip_mapping_children=True) for k in INDEX_MAP.keys()}
+
+    if is_admin:
+        fields_map.update({k: ADMIN_INDEX_MAP[k]().fields(skip_mapping_children=True) for k in ADMIN_INDEX_MAP.keys()})
+
+    return fields_map

howler/helper/workflow.py

@@ -0,0 +1,110 @@
+from typing import Callable, Optional, TypedDict, Union
+
+from howler.common.exceptions import HowlerException
+from howler.datastore.collection import ESCollection
+from howler.datastore.operations import OdmUpdateOperation
+
+
+class WorkflowException(HowlerException):
+    "Exception for errors caused during processing of a workflow"
+
+
+class Transition(TypedDict):
+    """Typed Dict outlining the propertyies of a valid transition object"""
+
+    source: Optional[Union[str, list[str]]]
+    transition: str
+    dest: Optional[str]
+    actions: list[Callable[..., list[OdmUpdateOperation]]]
+
+
+def validate_transition(transition: Transition):
+    "Ensure the given transition is valid"
+    return bool(
+        transition
+        # We want to check if a source is provided. If it is, it must have a value
+        # If it isn't, we'll allow this transition from any status
+        and ("source" not in transition or transition["source"] != "")
+        and transition["transition"]
+        # We want to check if a destination is provided. If it is, it must have a value
+        # If it isn't, we won't change the status of the hit
+        and ("dest" not in transition or transition["dest"] != "")
+        and isinstance(transition["actions"], list)
+        and all(callable(a) for a in transition["actions"])
+    )
+
+
+class Workflow:
+    """A simple state-like machine that generates OdmUpdateOperations on a given transition
+
+    NOTE: This does not keep track of state, it merely provides the update operations of a transition.
+    """
+
+    def __init__(self, status_prop: str, transitions: list[Transition]):
+        self.status_prop = status_prop
+
+        if any(not validate_transition(t) for t in transitions):
+            raise WorkflowException("One or more transitions provided were invalid.")
+
+        self.transitions = {}
+        identifiers = []
+        for t in transitions:
+            if t.get("source", False) and isinstance(t["source"], list):
+                for s in t["source"]:
+                    self.transitions[f'{s}{t["transition"]}'] = t
+                    identifiers.append(f'{s}{t["transition"]}{t.get("dest", None) or ""}')
+            else:
+                self.transitions[f'{t.get("source", "") or ""}{t["transition"]}'] = t
+                identifiers.append(f'{t.get("source", "") or ""}{t["transition"]}{t.get("dest", "") or ""}')
+
+        if len(set(identifiers)) != len(identifiers):
+            raise WorkflowException("There are duplicate transitions (same source, transition and dest values).")
+
+    def transition(self, current_status: str, transition: str, **kwargs) -> list[OdmUpdateOperation]:
+        "Generate a list of ODM updates based on the current status and a given transition step"
+        _transition: Optional[Transition] = self.transitions.get(
+            f"{current_status}{transition}", self.transitions.get(transition, None)
+        )
+        if not _transition:
+            raise WorkflowException(f"Current status '{current_status}' does not allow the '{transition}' transition.")
+
+        # Check if we can actually perform this transition
+        source = _transition.get("source")
+        if source and (isinstance(source, list) and current_status not in source) and current_status != source:
+            raise WorkflowException(f"Current status '{current_status}' does not allow the '{transition}' transition.")
+
+        updates_dict: dict[str, OdmUpdateOperation] = {}
+
+        for action in _transition.get("actions", []):
+            for update in action(transition=_transition, **kwargs):
+                # Check if an update already exists for this property and if it's value is different
+                if updates_dict.get(update.key) and updates_dict[update.key].value != update.value:
+                    raise WorkflowException(
+                        f"Transition {transition} attempted to update the same property {update.key} with \
+                            different values."
+                    )
+
+                updates_dict[update.key] = update
+
+        if self.status_prop not in updates_dict and _transition.get("dest", False):
+            updates_dict[self.status_prop] = OdmUpdateOperation(
+                ESCollection.UPDATE_SET,
+                self.status_prop,
+                _transition.get("dest"),
+            )
+
+        self.current_status = _transition.get("dest", current_status)
+
+        return list(updates_dict.values())
+
+    def get_transitions(self, current_status: str):
+        "Get a list of all given transitions"
+        return list(
+            set(
+                [
+                    t["transition"]
+                    for t in self.transitions.values()
+                    if (t["source"] and current_status in t["source"]) or not t["source"]
+                ]
+            )
+        )