howler-api 4.1.0.dev1616__tar.gz → 4.1.0.dev1624__tar.gz

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.1.0.dev1616
+Version: 4.1.0.dev1624
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/howler/api/v1/dossier.py

@@ -93,6 +93,26 @@ def create_dossier(**kwargs):
         return internal_error(err="An unknown error occured when creating the dossier.")
 
 
+@generate_swagger_docs()
+@dossier_api.route("/groups", methods=["GET"])
+@api_login(required_priv=["R"])
+def get_pivot_groups(user: User, **kwargs):
+    """Suggest existing pivot group paths matching a prefix
+
+    Variables:
+    None
+
+    Optional Arguments:
+    prefix => A case-insensitive prefix to filter suggestions by
+
+    Result Example:
+    [
+        "network/dns"   # Up to 25 matching group paths, used to autocomplete the pivot group field
+    ]
+    """
+    return ok(dossier_service.get_pivot_groups(request.args.get("prefix", "", type=str), username=user.uname))
+
+
 @generate_swagger_docs()
 @dossier_api.route("/<id>", methods=["GET"])
 @api_login(required_priv=["R"])

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/howler/odm/base.py

@@ -32,6 +32,7 @@ from howler.common import loader
 from howler.common.exceptions import HowlerKeyError, HowlerNotImplementedError, HowlerTypeError, HowlerValueError
 from howler.common.net import is_valid_domain, is_valid_ip
 from howler.odm.howler_enum import HowlerEnum
+from howler.utils.compat import Self
 from howler.utils.compat import StrEnum as PyStrEnum
 from howler.utils.dict_utils import flatten, recursive_update
 from howler.utils.isotime import now_as_iso
@@ -284,12 +285,21 @@ class Keyword(_Field):
     Examples: file hashes, service names, document ids
     """
 
-    def check(self, value, context=[], **kwargs):
+    def __init__(self, *args, coerce=True, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.coerce = coerce
+
+    def check(self: Self, value: Any | None, context=[], **kwargs):
         # We have a special case for bytes here due to how often strings and bytes
         # get mixed up in python apis
         if self.optional and value is None:
             return None
 
+        if not (self.coerce or isinstance(value, str)):
+            raise HowlerValueError(
+                f"[{'.'.join(context) or self.name}] Provided value is not a string, and coerce is set to false"
+            )
+
         if isinstance(value, bytes):
             raise HowlerValueError(f"[{'.'.join(context) or self.name}] Keyword doesn't accept bytes values")
 
@@ -422,7 +432,7 @@ class IP(Keyword):
         super().__init__(*args, **kwargs)
         self.validation_regex = re.compile(IP_ONLY_REGEX)
 
-    def check(self, value, context=[], **kwargs):
+    def check(self: Self, value: str, context: list[str] = [], **kwargs):
         if not value:
             return None
 
@@ -465,7 +475,7 @@ class Email(Keyword):
         super().__init__(*args, **kwargs)
         self.validation_regex = re.compile(EMAIL_REGEX)
 
-    def check(self, value, context=[], **kwargs):
+    def check(self: Self, value: str, context: list[str] = [], **kwargs):
         if not value:
             return None
 
@@ -489,7 +499,7 @@ class URI(Keyword):
         super().__init__(*args, **kwargs)
         self.validation_regex = re.compile(FULL_URI)
 
-    def check(self, value, context=[], **kwargs):
+    def check(self: Self, value: str, context: list[str] = [], **kwargs):
         if not value:
             return None
 

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/howler/odm/helper.py

@@ -1,9 +1,8 @@
 import json
-import random
 from datetime import datetime, timedelta
 from hashlib import md5
 from math import ceil
-from random import choice, randint, sample
+from random import choice, choices, gauss, randint, random, sample
 from typing import Any, cast
 
 from howler.common.logging import get_logger
@@ -55,13 +54,13 @@ def generate_useful_hit(  # noqa: C901
     if CLASSIFICATION.enforce:
         hit.classification = CLASSIFICATION.UNRESTRICTED
 
-    rand_seed = random.random()
+    rand_seed = random()
 
     timestamp = datetime.now() - timedelta(
         days=round(rand_seed * 30),
-        hours=min(max(round(random.gauss(14, 3)), 0), 23),
-        minutes=random.randint(0, 59),
-        seconds=random.randint(0, 59),
+        hours=min(max(round(gauss(14, 3)), 0), 23),
+        minutes=randint(0, 59),
+        seconds=randint(0, 59),
     )
 
     hit.event.created = timestamp.isoformat() + "Z"
@@ -251,7 +250,7 @@ def generate_useful_hit(  # noqa: C901
                     "icon": app["name"],
                 }
             )
-            for app in random.choices(APPS, k=5)
+            for app in choices(APPS, k=5)
         )
     except IndexError:
         pass
@@ -362,13 +361,13 @@ def generate_useful_event(  # noqa: C901
     "Create a random, useful/cogent event for synthetic data"
     event: Event = random_model_obj(cast(Model, Event))
 
-    rand_seed = random.random()
+    rand_seed = random()
 
     timestamp = datetime.now() - timedelta(
         days=round(rand_seed * 30),
-        hours=min(max(round(random.gauss(14, 3)), 0), 23),
-        minutes=random.randint(0, 59),
-        seconds=random.randint(0, 59),
+        hours=min(max(round(gauss(14, 3)), 0), 23),
+        minutes=randint(0, 59),
+        seconds=randint(0, 59),
     )
 
     event.event.created = timestamp.isoformat() + "Z"
@@ -518,6 +517,7 @@ def generate_useful_dossier(users: list[User]) -> Dossier:
                     "label": {"en": get_random_word(), "fr": get_random_word()},
                     "value": f"https://google.com/search?q={{{{test{i}}}}} and {{{{custom_value}}}}",
                     "format": "link",
+                    "group": choice([None, "parent/child", "test/group"]),
                     "mappings": [
                         {"key": f"test{i}", "field": choice(list(Hit.flat_fields().keys()))},
                         {"key": "custom_value", "field": "custom", "custom_value": get_random_word()},

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/howler/odm/models/pivot.py

@@ -1,7 +1,8 @@
 # mypy: ignore-errors
-from typing import Optional
+from typing import Any, Optional
 
 from howler import odm
+from howler.common.exceptions import HowlerValueError
 from howler.odm.models.localized_label import LocalizedLabel
 
 
@@ -21,14 +22,35 @@ class Mapping(odm.Model):
     )
 
 
+def validate_group(group: str | Any) -> None:
+    """Validate a slash-separated pivot group path."""
+    if group is None:
+        return
+
+    if group == "":
+        raise HowlerValueError("Group cannot be an empty string")
+
+    if not isinstance(group, str):
+        raise HowlerValueError('Data "group" should be a slash-separated string.')
+
+    if any(not section for section in group.split("/")):
+        raise HowlerValueError("A group must consist of a relative path with no empty segments (e.g. a/b/c)")
+
+
 @odm.model(
     index=False,
     store=True,
     description="The dossier object stores individual tabs/fields for a given alert.",
 )
 class Pivot(odm.Model):
-    icon: Optional[str] = odm.Text(
-        description="An optional icon to use in the tab display for this dossier.", optional=True
+    icon: str = odm.Text(
+        description="An optional icon to use in the tab display for this dossier.", default="material-symbols:link"
+    )
+    group: str | None = odm.Keyword(
+        description="The folder path this pivot should be presented in when viewed in the UI.",
+        optional=True,
+        index=True,
+        coerce=False,
     )
     label: LocalizedLabel = odm.Compound(LocalizedLabel, description="Labels for the pivot in the UI.")
     value: str = odm.Keyword(description="The link/plugin information to pivot on.")
@@ -38,3 +60,11 @@ class Pivot(odm.Model):
         default=[],
         description="A list of the mappings to use when activating a pivot.",
     )
+
+    def __init__(self, data: dict = None, *args, **kwargs):
+        super().__init__(data, *args, **kwargs)
+
+        validate_group(self.group)
+
+        if len(self.mappings) != len({mapping.key for mapping in self.mappings}):
+            raise HowlerValueError("One of your pivots has duplicate keys set.")

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/howler/odm/random_data.py

@@ -36,10 +36,12 @@ from howler.odm.base import Keyword
 from howler.odm.helper import generate_useful_case, generate_useful_dossier, generate_useful_event, generate_useful_hit
 from howler.odm.models.action import Action
 from howler.odm.models.analytic import Analytic, Comment, Notebook, TriageOptions
+from howler.odm.models.dossier import Dossier
 from howler.odm.models.ecs.event import EVENT_CATEGORIES
 from howler.odm.models.hit import Hit
 from howler.odm.models.howler_data import Assessment, Escalation, Scrutiny, Status
 from howler.odm.models.overview import Overview
+from howler.odm.models.pivot import Pivot
 from howler.odm.models.template import Template
 from howler.odm.models.user import User
 from howler.odm.models.view import View
@@ -881,6 +883,7 @@ def wipe_actions(ds: HowlerDatastore):
 def create_dossiers(ds: HowlerDatastore, num_dossiers: int = 5):
     "Create random dossiers"
     users = ds.user.search("*:*")["items"]
+    grouped_paths = ("network", "network/dns", "network/http")
     for index in range(num_dossiers):
         dossier = generate_useful_dossier(users)
         ownership_data = _get_ownership_data(users, dossier.owner)
@@ -890,8 +893,56 @@ def create_dossiers(ds: HowlerDatastore, num_dossiers: int = 5):
             dossier.type = "personal"
         elif index == 1:
             dossier.type = "global"
+
+        # Keep one dossier's random pivot data intact, while ensuring the mock data
+        # always includes shared and nested groups for the grouped-pivot UI.
+        if index > 0:
+            grouped_path = grouped_paths[(index - 1) % len(grouped_paths)]
+            for pivot in dossier.pivots:
+                pivot.group = grouped_path
+
         ds.dossier.save(dossier.dossier_id, dossier)
 
+    password_checker_dossier = Dossier(
+        {
+            "title": "Password Checker Pivot Folders",
+            "query": 'howler.analytic:"Password Checker"',
+            "type": "global",
+            **_get_ownership_data(users, "admin"),
+            "leads": [],
+        }
+    )
+    pivot_groups = [
+        None,
+        "credentials",
+        "credentials/hash",
+        "credentials/metadata",
+        "validation",
+        "validation/length",
+        "validation/complexity",
+        "validation/reuse",
+        "exposure",
+        "exposure/breach",
+        "exposure/darkweb",
+        "exposure/thirdparty",
+        "recommendations",
+        "recommendations/rotation",
+        "recommendations/monitoring",
+    ]
+    for index, pivot_group in enumerate(pivot_groups, start=1):
+        password_checker_dossier.pivots.append(
+            Pivot(
+                {
+                    "label": {"en": f"Password Check {index}", "fr": f"Verification mot de passe {index}"},
+                    "value": "https://password-checker.example.test/{{pivot_value}}",
+                    "format": "link",
+                    "group": pivot_group,
+                    "mappings": [{"key": "pivot_value", "field": "howler.id"}],
+                }
+            )
+        )
+
+    ds.dossier.save(password_checker_dossier.dossier_id, password_checker_dossier)
     ds.dossier.commit()
 
 

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/howler/services/dossier_service.py

@@ -9,13 +9,20 @@ from typing import Any, Literal, Optional, cast, overload
 
 from mergedeep.mergedeep import merge
 
-from howler.common.exceptions import ForbiddenException, HowlerException, InvalidDataException, NotFoundException
+from howler.common.exceptions import (
+    ForbiddenException,
+    HowlerException,
+    HowlerValueError,
+    InvalidDataException,
+    NotFoundException,
+)
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.datastore.exceptions import SearchException
 from howler.odm.models.dossier import Dossier
 from howler.odm.models.user import User
 from howler.services import lucene_service
+from howler.utils.str_utils import sanitize_lucene_query
 
 logger = get_logger(__file__)
 
@@ -28,6 +35,12 @@ PERMITTED_KEYS = {
     "type",
 }
 
+# Cap the number of group suggestions returned, so the endpoint can't be used to dump the entire group list
+MAX_GROUP_SUGGESTIONS = 25
+GROUP_SUGGESTION_PAGE_SIZE = 100
+# Group suggestions are a convenience feature, so bound aggregation work for broad prefixes.
+MAX_GROUP_SUGGESTION_PAGES = 5
+
 
 def exists(dossier_id: str) -> bool:
     """Check if a dossier exists in the datastore.
@@ -140,11 +153,6 @@ def create_dossier(  # noqa: C901
 
         dossier = Dossier(dossier_data)
 
-        # Validate pivot configurations to ensure no duplicate mapping keys
-        for pivot in dossier.pivots:
-            if len(pivot.mappings) != len(set(mapping.key for mapping in pivot.mappings)):
-                raise InvalidDataException("One of your pivots has duplicate keys set.")
-
         # Ensure the owner is set to the current user.
         dossier.owner = username
 
@@ -208,14 +216,6 @@ def update_dossier(  # noqa: C901
             "You cannot update a dossier that is not owned by you, or you are not an administrator of."
         )
 
-    # Validate pivot configurations if they're being updated
-    # Ensure no duplicate mapping keys exist within any pivot
-    if "pivots" in dossier_data:
-        for pivot in dossier_data["pivots"]:
-            mappings = pivot.get("mappings") or []
-            if len(mappings) != len(set(mapping.get("key") for mapping in mappings)):
-                raise InvalidDataException("One of your pivots has duplicate keys set.")
-
     try:
         # Validate the Lucene query if it's being updated
         if "query" in dossier_data:
@@ -231,6 +231,8 @@ def update_dossier(  # noqa: C901
     except SearchException:
         # Handle invalid Lucene query syntax
         raise InvalidDataException("You must use a valid query when updating a dossier.")
+    except HowlerValueError as e:
+        raise InvalidDataException(str(e), cause=e) from e
     except (HowlerException, TypeError) as e:
         # Log the error for debugging purposes
         logger.exception("Error when updating dossier.")
@@ -238,6 +240,53 @@ def update_dossier(  # noqa: C901
         raise InvalidDataException("We were unable to update the dossier.", cause=e) from e
 
 
+def get_pivot_groups(prefix: str, username: str) -> list[str]:
+    """Suggest existing pivot group paths matching a prefix, so users can reuse groups already in use.
+
+    Args:
+        prefix: Case-insensitive prefix to filter suggestions by.
+        username: The requesting user, used to scope which dossiers are visible.
+
+    Returns:
+        Up to MAX_GROUP_SUGGESTIONS unique matching group paths from the visible dossier search, sorted alphabetically.
+        Suggestions are best-effort and scan up to MAX_GROUP_SUGGESTION_PAGES matching aggregation pages.
+    """
+    groups: set[str] = set()
+    lowered_prefix: str = (prefix or "").lower()
+    after: dict[str, str] | None = None
+
+    for _ in range(MAX_GROUP_SUGGESTION_PAGES):
+        composite: dict[str, Any] = {
+            "size": GROUP_SUGGESTION_PAGE_SIZE,
+            "sources": [{"group": {"terms": {"field": "pivots.group"}}}],
+        }
+        if after:
+            composite["after"] = after
+
+        group_filter: dict[str, Any] = {"match_all": {}}
+        if prefix:
+            group_filter = {"prefix": {"pivots.group": {"value": prefix, "case_insensitive": True}}}
+
+        result = datastore().dossier.search(
+            f'(type:global OR owner:("{sanitize_lucene_query(username)}" OR none))',
+            rows=0,
+            aggregations=[("pivot_groups", {"filter": group_filter, "aggs": {"groups": {"composite": composite}}})],
+        )
+        aggregation = result["aggregations"]["pivot_groups"]["groups"]
+
+        for bucket in aggregation["buckets"]:
+            group = bucket["key"]["group"]
+            if not group.lower().startswith(lowered_prefix):
+                continue
+            groups.add(group)
+
+        after = aggregation.get("after_key")
+        if not after:
+            break
+
+    return sorted(groups)[:MAX_GROUP_SUGGESTIONS]
+
+
 def get_matching_dossiers(
     hit: dict[str, Any], dossiers: list[dict[str, Any]] | None = None, username: str | None = None
 ):

{howler_api-4.1.0.dev1616 → howler_api-4.1.0.dev1624}/pyproject.toml

@@ -76,9 +76,101 @@ module = "requests"
 ignore_missing_imports = true
 
 [tool.pyright]
+exclude = [
+    "**/build",
+    "**/.venv",
+    "**/node_modules",
+    "**/__pycache__",
+    "**/.*",
+]
+ignore = [
+    "**/build",
+    "**/.venv",
+]
+reportMissingImports = "error"
+reportOptionalMemberAccess = "error"
+strictDictionaryInference = true
+strictListInference = true
 venvPath = "."
 venv = ".venv"
 
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportArgumentType = "none"
+reportAssignmentType = "none"
+root = "howler/odm/models"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportArgumentType = "information"
+reportAttributeAccessIssue = "information"
+reportOptionalCall = "information"
+root = "howler/datastore"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportOptionalMemberAccess = "information"
+reportPossiblyUnboundVariable = "information"
+root = "howler/odm/helper.py"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportPossiblyUnboundVariable = "information"
+root = "howler/odm/random_data.py"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportArgumentType = "information"
+reportAttributeAccessIssue = "information"
+reportIncompatibleMethodOverride = "information"
+reportOperatorIssue = "information"
+reportOptionalMemberAccess = "information"
+reportPossiblyUnboundVariable = "information"
+root = "howler/odm/base.py"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportAttributeAccessIssue = "information"
+root = "howler/security/login.py"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportArgumentType = "information"
+reportOperatorIssue = "information"
+reportOptionalMemberAccess = "information"
+reportPossiblyUnboundVariable = "information"
+root = "howler/helper/ws.py"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportArgumentType = "information"
+reportIncompatibleMethodOverride = "information"
+reportReturnType = "information"
+root = "howler/remote/datatypes"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportArgumentType = "none"
+root = "howler/common/classification.py"
+
+[[tool.pyright.executionEnvironments]]
+extraPaths = ["."]
+reportArgumentType = "none"
+reportAttributeAccessIssue = "none"
+reportCallIssue = "none"
+reportGeneralTypeIssues = "none"
+reportInvalidTypeForm = "none"
+reportOperatorIssue = "none"
+reportOptionalIterable = "none"
+reportOptionalMemberAccess = "none"
+reportOptionalOperand = "none"
+reportOptionalSubscript = "none"
+reportPossiblyUnboundVariable = "none"
+reportReturnType = "none"
+reportTypedDictNotRequiredAccess = "none"
+reportUnusedExpression = "none"
+root = "test"
+
 [tool.ruff]
 line-length = 120
 indent-width = 4
@@ -152,7 +244,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.1.0.dev1616"
+version = "4.1.0.dev1624"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",