howler-api 2.11.0.dev185__tar.gz → 2.11.0.dev217__tar.gz

{howler_api-2.11.0.dev185 → howler_api-2.11.0.dev217}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: howler-api
-Version: 2.11.0.dev185
+Version: 2.11.0.dev217
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-2.11.0.dev185 → howler_api-2.11.0.dev217}/howler/api/v1/auth.py

@@ -131,7 +131,7 @@ def add_apikey(**kwargs):  # noqa: C901
         key_name = apikey_data["name"] if "I" not in privs else f"impersonate_{apikey_data['name']}"
 
         new_key = {
-            "password": bcrypt.encrypt(random_pass),
+            "password": bcrypt.hash(random_pass),
             "agents": apikey_data.get("agents", []),
             "acl": privs,
         }

{howler_api-2.11.0.dev185 → howler_api-2.11.0.dev217}/howler/api/v1/dossier.py

@@ -1,15 +1,6 @@
 from flask import request
 
-from howler.api import (
-    bad_request,
-    created,
-    forbidden,
-    internal_error,
-    make_subapi_blueprint,
-    no_content,
-    not_found,
-    ok,
-)
+from howler.api import bad_request, created, forbidden, internal_error, make_subapi_blueprint, no_content, not_found, ok
 from howler.common.exceptions import ForbiddenException, HowlerException, InvalidDataException, NotFoundException
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
@@ -17,7 +8,7 @@ from howler.common.swagger import generate_swagger_docs
 from howler.odm.models.dossier import Dossier
 from howler.odm.models.user import User
 from howler.security import api_login
-from howler.services import dossier_service, lucene_service
+from howler.services import dossier_service
 
 SUB_API = "dossier"
 dossier_api = make_subapi_blueprint(SUB_API, api_version=1)
@@ -141,29 +132,14 @@ def get_dossier_for_hit(id: str, user: User, **kwargs):
     """
     storage = datastore()
     try:
-        response = storage.hit.search(f"howler.id:{id}", rows=1)
+        response = storage.hit.search(f"howler.id:{id}", rows=1, as_obj=False)
 
         if response["total"] < 1:
             return not_found(err="Hit does not exist.")
 
         hit = response["items"][0]
 
-        results: list[Dossier] = storage.dossier.search(
-            "dossier_id:*",
-            as_obj=True,
-            rows=1000,
-        )["items"]
-
-        matching_dossiers: list[Dossier] = []
-        for dossier in results:
-            if dossier.query is None:
-                matching_dossiers.append(dossier)
-                continue
-
-            if lucene_service.match(dossier.query, hit.as_primitives()):
-                matching_dossiers.append(dossier)
-
-        return ok(matching_dossiers)
+        return ok(dossier_service.get_matching_dossiers(hit))
     except ValueError as e:
         return bad_request(err=str(e))
 

{howler_api-2.11.0.dev185 → howler_api-2.11.0.dev217}/howler/api/v1/hit.py

@@ -17,11 +17,7 @@ from howler.api import (
     ok,
 )
 from howler.api.v1.utils.etag import add_etag
-from howler.common.exceptions import (
-    HowlerException,
-    HowlerValueError,
-    InvalidDataException,
-)
+from howler.common.exceptions import HowlerException, HowlerValueError, InvalidDataException
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.common.swagger import generate_swagger_docs
@@ -252,7 +248,7 @@ def validate_hits(**kwargs):
 @generate_swagger_docs()
 @hit_api.route("/<id>", methods=["GET"])
 @api_login(audit=False, required_priv=["R"])
-@add_etag(getter=hit_service.get_hit, check_if_match=False)
+@add_etag(getter=hit_service.get_hit)
 def get_hit(id: str, server_version: str, **kwargs):
     """Get a hit.
 
@@ -265,11 +261,19 @@ def get_hit(id: str, server_version: str, **kwargs):
     Result Example:
     https://github.com/CybercentreCanada/howler-api/blob/main/howler/odm/models/hit.py
     """
-    hit = cast(Optional[Hit], kwargs.get("cached_hit"))
+    hit = cast(Optional[Any], kwargs.get("cached_hit"))
 
     if not hit:
         return not_found(err="Hit %s does not exist" % id)
 
+    if "metadata" in request.args:
+        metadata = (request.args.get("metadata", type=str) or "").split(",")
+
+        hit = hit.as_primitives()
+
+        if len(metadata) > 0:
+            hit_service.augment_metadata(hit, metadata, kwargs["user"])
+
     return ok(hit), server_version
 
 

{howler_api-2.11.0.dev185 → howler_api-2.11.0.dev217}/howler/api/v1/search.py

@@ -14,6 +14,7 @@ from howler.common.swagger import generate_swagger_docs
 from howler.datastore.exceptions import SearchException
 from howler.helper.search import get_collection, get_default_sort, has_access_control, list_all_fields
 from howler.security import api_login
+from howler.services import hit_service
 
 SUB_API = "search"
 search_api = make_subapi_blueprint(SUB_API, api_version=1)
@@ -73,16 +74,18 @@ def search(index, **kwargs):
     timeout             =>   Maximum execution time (ms)
     use_archive         =>   Allow access to the datastore achive (Default: False)
     track_total_hits    =>   Track the total number of query matches, instead of stopping at 10000 (Default: False)
+    metadata            =>   A list of additional features to be added to the result alongside the raw results
 
     Data Block:
     # Note that the data block is for POST requests only!
-    {"query": "query",     # Query to search for
-     "offset": 0,          # Offset in the results
-     "rows": 100,          # Max number of results
-     "sort": "field asc",  # How to sort the results
-     "fl": "id,score",     # List of fields to return
-     "timeout": 1000,      # Maximum execution time (ms)
-     "filters": ['fq']}    # List of additional filter queries limit the data
+    {"query": "query",          # Query to search for
+     "offset": 0,               # Offset in the results
+     "rows": 100,               # Max number of results
+     "sort": "field asc",       # How to sort the results
+     "fl": "id,score",          # List of fields to return
+     "timeout": 1000,           # Maximum execution time (ms)
+     "filters": ['fq'],         # List of additional filter queries limit the data
+     "metadata": ["dossiers"]}  # List of additional features to add to the search
 
 
     Result Example:
@@ -108,7 +111,7 @@ def search(index, **kwargs):
         "deep_paging_id",
         "track_total_hits",
     ]
-    multi_fields = ["filters"]
+    multi_fields = ["filters", "metadata"]
     boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
@@ -132,7 +135,13 @@ def search(index, **kwargs):
         return bad_request(err="There was no search query.")
 
     try:
-        return ok(collection().search(query, **params))
+        metadata = params.pop("metadata", [])
+        result = collection().search(query, **params)
+
+        if index == "hit" and len(metadata) > 0:
+            hit_service.augment_metadata(result["items"], metadata, user)
+
+        return ok(result)
     except (SearchException, BadRequestError) as e:
         return bad_request(err=f"SearchException: {e}")
 

{howler_api-2.11.0.dev185 → howler_api-2.11.0.dev217}/howler/api/v1/user.py

@@ -145,7 +145,7 @@ def add_user_account(username, **_):
 @generate_swagger_docs()
 @user_api.route("/<username>", methods=["GET"])
 @api_login(audit=False, required_priv=["R"])
-@add_etag(getter=user_service.get_user, check_if_match=False)
+@add_etag(getter=user_service.get_user, check_if_match=True)
 def get_user_account(username: str, server_version: Optional[str] = None, **kwargs):
     """Load the user account information.
 
@@ -327,7 +327,7 @@ def get_user_avatar(username, **_):
         resp.headers["ETag"] = sha256(avatar.encode("utf-8")).hexdigest()
         return resp
     else:
-        return not_found(err="No avatar for specified user")
+        return no_content()
 
 
 @generate_swagger_docs()

howler_api-2.11.0.dev217/howler/api/v1/utils/etag.py

@@ -0,0 +1,84 @@
+"""ETag utility module for handling HTTP ETags in Flask responses.
+
+ETags (Entity Tags) are HTTP headers used for web cache validation and conditional requests.
+They help optimize performance by allowing clients to cache responses and only fetch
+new data when the resource has actually changed.
+"""
+
+import functools
+import re
+
+from flask import Response, request
+
+from howler.api import not_modified
+
+
+def add_etag(getter, check_if_match=True):
+    """Decorator to add ETag handling to a Flask response.
+
+    This decorator implements HTTP ETag functionality for API endpoints, enabling:
+    - Conditional requests using If-Match headers
+    - Cache validation to prevent unnecessary data transfers
+    - Version tracking for resources
+
+    Args:
+        getter: Function that retrieves the object and its version
+        check_if_match (bool): Whether to check If-Match headers for conditional requests
+
+    Returns:
+        Decorated function with ETag support
+    """
+
+    def wrapper(f):
+        """Inner wrapper function that applies ETag functionality to the decorated function."""
+
+        @functools.wraps(f)
+        def generate_etag(*args, **kwargs):
+            """Generate and handle ETags for the HTTP response."""
+            # Retrieve the object and its version using the provided getter function
+            # The getter should return (object, version) tuple
+            obj, version = getter(
+                kwargs.get("id", kwargs.get("username", None)),
+                as_odm=True,
+                version=True,
+            )
+
+            # Handle conditional requests with If-Match header
+            # If the client's version matches the current version and it's a GET request
+            # without metadata parameter, return 304 Not Modified to save bandwidth
+            if (
+                check_if_match
+                and "If-Match" in request.headers
+                and request.headers["If-Match"] == version
+                and request.method == "GET"
+                and "metadata" not in request.args
+            ):
+                return not_modified()
+
+            # Extract the resource type from the API path and create a cache key
+            # e.g., "/api/v1/users/123" becomes "cached_users"
+            key = re.sub(r"^\/api\/v\d+\/(\w+)\/.+$", r"cached_\1", request.path)
+            kwargs[key] = obj
+
+            # Call the original function with the cached object and version
+            values = f(*args, server_version=version, **kwargs)
+
+            # Handle different return value formats from the decorated function
+            # If there is only one return, it's just the response
+            if isinstance(values, Response):
+                # Only add ETag header for successful responses (not 409 Conflict or 400 Bad Request)
+                if values.status_code != 409 and values.status_code != 400:
+                    values.headers["ETag"] = version
+                return values
+
+            # If there are two returns, it's the response and the new version
+            # This happens when the function modifies the resource and returns an updated version
+            else:
+                if values[0].status_code != 409 and values[0].status_code != 400:
+                    # Add the new ETag version to successful responses
+                    values[0].headers["ETag"] = values[1]
+                return values[0]
+
+        return generate_etag
+
+    return wrapper

{howler_api-2.11.0.dev185 → howler_api-2.11.0.dev217}/howler/odm/helper.py

@@ -143,8 +143,9 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
     hit.howler.labels.mitigation = []
     hit.howler.labels.operation = []
     hit.howler.labels.threat = []
+    hit.howler.labels.tuning = []
 
-    label_type = ceil(rand_seed * 6)
+    label_type = ceil(rand_seed * 7)
     if label_type == 1:
         hit.howler.labels.campaign = ["Bad event 2023-07"]
     elif label_type == 2:
@@ -155,6 +156,8 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
         hit.howler.labels.mitigation = ["Blocked: google.com"]
     elif label_type == 5:
         hit.howler.labels.operation = ["OP_HOWLER"]
+    elif label_type == 6:
+        hit.howler.labels.tuning = ["Tune example"]
     else:
         hit.howler.labels.threat = ["Bad Mojo"]
 

howler_api-2.11.0.dev217/howler/services/dossier_service.py

@@ -0,0 +1,252 @@
+"""Dossier service module for managing security investigation dossiers.
+
+This module provides functionality for creating, updating, retrieving, and managing
+dossiers - collections of security alerts and investigation data organized by analysts.
+Dossiers can be personal (private to the creator) or global (shared with the team).
+"""
+
+from typing import Any, Optional, cast
+
+from mergedeep.mergedeep import merge
+
+from howler.common.exceptions import ForbiddenException, HowlerException, InvalidDataException, NotFoundException
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.datastore.exceptions import SearchException
+from howler.odm.models.dossier import Dossier
+from howler.odm.models.user import User
+from howler.services import lucene_service
+
+logger = get_logger(__file__)
+
+# Define which fields are allowed to be updated in a dossier, preventing unauthorized modification of sensitive fields
+PERMITTED_KEYS = {
+    "title",
+    "query",
+    "leads",
+    "pivots",
+    "type",
+    "owner",
+}
+
+
+def exists(dossier_id: str) -> bool:
+    """Check if a dossier exists in the datastore.
+
+    Args:
+        dossier_id: Unique identifier for the dossier
+
+    Returns:
+        True if the dossier exists, False otherwise
+    """
+    return datastore().dossier.exists(dossier_id)
+
+
+def get_dossier(
+    id: str,
+    as_odm: bool = False,
+    version: bool = False,
+) -> Dossier:
+    """Retrieve a dossier from the datastore.
+
+    Args:
+        id: Unique identifier for the dossier
+        as_odm: Whether to return as ODM object (True) or dictionary (False)
+        version: Whether to include version information in the response
+
+    Returns:
+        Dossier object or dictionary containing dossier data
+
+    Raises:
+        NotFoundException: If the dossier doesn't exist
+    """
+    return datastore().dossier.get_if_exists(key=id, as_obj=as_odm, version=version)
+
+
+def create_dossier(dossier_data: Optional[Any], username: str) -> Dossier:  # noqa: C901
+    """Create a new dossier in the datastore.
+
+    This function validates the input data, ensures the query is valid by testing it
+    against the hit collection, and creates a new dossier with the specified parameters.
+
+    Args:
+        dossier_data: Dictionary containing dossier configuration data
+        username: Username of the user creating the dossier
+
+    Returns:
+        Newly created Dossier object
+
+    Raises:
+        InvalidDataException: If data format is invalid, required fields are missing,
+                            or the query is invalid
+        HowlerException: If there's an error during dossier creation
+    """
+    # Validate input data format
+    if not isinstance(dossier_data, dict):
+        raise InvalidDataException("Invalid data format")
+
+    # Validate required fields for dossier creation
+    if "title" not in dossier_data:
+        raise InvalidDataException("You must specify a title when creating a dossier.")
+
+    if "query" not in dossier_data:
+        raise InvalidDataException("You must specify a query when creating a dossier.")
+
+    if "type" not in dossier_data:
+        raise InvalidDataException("You must specify a type when creating a dossier.")
+
+    storage = datastore()
+
+    try:
+        # Validate the Lucene query by attempting to search with it
+        # This ensures the query syntax is correct before saving the dossier
+        if query := dossier_data.get("query", None):
+            storage.hit.search(query)
+
+        if "owner" not in dossier_data:
+            dossier_data["owner"] = username
+
+        dossier = Dossier(dossier_data)
+
+        # Validate pivot configurations to ensure no duplicate mapping keys
+        for pivot in dossier.pivots:
+            if len(pivot.mappings) != len(set(mapping.key for mapping in pivot.mappings)):
+                raise InvalidDataException("One of your pivots has duplicate keys set.")
+
+        # Ensure the owner is set to the current user (security measure)
+        dossier.owner = username
+
+        # Save the dossier to the datastore
+        storage.dossier.save(dossier.dossier_id, dossier)
+
+        # Commit the transaction to persist changes
+        storage.dossier.commit()
+
+        return dossier
+    except SearchException:
+        # Handle invalid Lucene query syntax
+        raise InvalidDataException("You must use a valid query when creating a dossier.")
+    except HowlerException as e:
+        # Handle other application-specific errors
+        raise InvalidDataException(str(e))
+
+
+def update_dossier(dossier_id: str, dossier_data: dict[str, Any], user: User) -> Dossier:  # noqa: C901
+    """Update one or more properties of a dossier in the database.
+
+    This function enforces access control rules and validates data before updating.
+    Personal dossiers can only be updated by their owners or admins.
+    Global dossiers can only be updated by their owners or admins.
+
+    Args:
+        dossier_id: Unique identifier of the dossier to update
+        dossier_data: Dictionary containing fields to update
+        user: User object representing the requesting user
+
+    Returns:
+        Updated Dossier object
+
+    Raises:
+        NotFoundException: If the dossier doesn't exist
+        InvalidDataException: If invalid fields are provided or data is malformed
+        ForbiddenException: If user lacks permission to update the dossier
+    """
+    # Verify the dossier exists before attempting to update
+    if not exists(dossier_id):
+        raise NotFoundException(f"Dossier with id '{dossier_id}' does not exist.")
+
+    # Validate that only permitted fields are being updated
+    # This prevents unauthorized modification of sensitive fields
+    if set(dossier_data.keys()) - PERMITTED_KEYS:
+        raise InvalidDataException(f"Only {', '.join(PERMITTED_KEYS)} can be updated.")
+
+    storage = datastore()
+
+    # Retrieve the existing dossier for access control checks
+    existing_dossier: Dossier = get_dossier(dossier_id, as_odm=True)
+
+    # Enforce access control for personal dossiers
+    # Only the owner or admin users can modify personal dossiers
+    if existing_dossier.type == "personal" and existing_dossier.owner != user.uname and "admin" not in user.type:
+        raise ForbiddenException("You cannot update a personal dossier that is not owned by you.")
+
+    # Enforce access control for global dossiers
+    # Only the owner or admin users can modify global dossiers
+    if existing_dossier.type == "global" and existing_dossier.owner != user.uname and "admin" not in user.type:
+        raise ForbiddenException("Only the owner of a dossier and administrators can edit a global dossier.")
+
+    # Validate pivot configurations if they're being updated
+    # Ensure no duplicate mapping keys exist within any pivot
+    if "pivots" in dossier_data:
+        for pivot in dossier_data["pivots"]:
+            if len(pivot["mappings"]) != len(set(mapping["key"] for mapping in pivot["mappings"])):
+                raise InvalidDataException("One of your pivots has duplicate keys set.")
+
+    try:
+        # Validate the Lucene query if it's being updated
+        if "query" in dossier_data:
+            # Test the query against the hit index to ensure it's valid
+            storage.hit.search(dossier_data["query"])
+
+        # Merge the new data with existing dossier data
+        new_data = Dossier(merge({}, existing_dossier.as_primitives(), dossier_data))
+
+        storage.dossier.save(dossier_id, new_data)
+
+        # Commit the transaction to persist changes
+        storage.dossier.commit()
+
+        return new_data
+    except SearchException:
+        # Handle invalid Lucene query syntax
+        raise InvalidDataException("You must use a valid query when updating a dossier.")
+    except (HowlerException, TypeError) as e:
+        # Log the error for debugging purposes
+        logger.exception("Error when updating dossier.")
+        # Provide a user-friendly error message while preserving the original exception
+        raise InvalidDataException("We were unable to update the dossier.", cause=e) from e
+
+
+def get_matching_dossiers(hit: dict[str, Any], dossiers: Optional[list[dict[str, Any]]] = None):
+    """Get a list of dossiers that match a specific security alert/hit.
+
+    This function evaluates each dossier's query against the provided hit data
+    to determine which dossiers are relevant to the security event.
+
+    Args:
+        hit: Dictionary containing security alert/hit data to match against
+        dossiers: Optional list of dossiers to check. If None, all dossiers
+                 will be retrieved from the datastore
+
+    Returns:
+        List of dossier dictionaries that match the provided hit
+
+    Note:
+        This function uses Lucene query matching to determine relevance.
+        Dossiers with no query are assumed to match all hits.
+    """
+    # Retrieve all dossiers if none provided
+    if dossiers is None:
+        dossiers: list[dict[str, Any]] = datastore().dossier.search(
+            "dossier_id:*",
+            as_obj=False,
+            # TODO: Eventually implement caching here
+            rows=1000,
+        )["items"]
+
+    matching_dossiers: list[dict[str, Any]] = []
+
+    # Evaluate each dossier against the hit data
+    for dossier in cast(list[dict[str, Any]], dossiers):
+        # Dossiers without queries match all hits by default
+        # This allows for catch-all dossiers that collect all security events
+        if "query" not in dossier or dossier["query"] is None:
+            matching_dossiers.append(dossier)
+            continue
+
+        # Use Lucene service to check if the hit matches the dossier's query
+        # This determines if the security event is relevant to this investigation
+        if lucene_service.match(dossier["query"], hit):
+            matching_dossiers.append(dossier)
+
+    return matching_dossiers