howler-api 4.0.0.dev740__tar.gz → 4.0.0.dev799__tar.gz

{howler_api-4.0.0.dev740 → howler_api-4.0.0.dev799}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev740
+Version: 4.0.0.dev799
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -8,7 +8,7 @@ Author: Canadian Centre for Cyber Security
 Author-email: howler@cyber.gc.ca
 Maintainer: Matthew Rafuse
 Maintainer-email: matthew.rafuse@cyber.gc.ca
-Requires-Python: >=3.9.17,<4.0.0
+Requires-Python: >=3.10,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -21,10 +21,10 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Requires-Dist: apscheduler (==3.11.2)
 Requires-Dist: authlib (>=1.6.0,<2.0.0)
+Requires-Dist: azure-monitor-opentelemetry (>=1.8.7,<2.0.0)
 Requires-Dist: bcrypt (==4.3.0)
 Requires-Dist: chardet (==5.2.0)
 Requires-Dist: chevron (==0.14.0)
-Requires-Dist: elastic-apm[flask] (>=6.22.0,<7.0.0)
 Requires-Dist: elasticsearch (==8.19.3)
 Requires-Dist: flasgger (>=0.9.7.1,<0.10.0.0)
 Requires-Dist: flask (==3.1.3)
@@ -33,6 +33,9 @@ Requires-Dist: gevent (>=25.9.1,<26.0.0)
 Requires-Dist: gunicorn (==23.0.0)
 Requires-Dist: luqum (>=1.0.0,<2.0.0)
 Requires-Dist: mergedeep (>=1.3.4,<2.0.0)
+Requires-Dist: opentelemetry-exporter-otlp-proto-http (==1.40.0)
+Requires-Dist: opentelemetry-instrumentation-flask (==0.61b0)
+Requires-Dist: opentelemetry-sdk (==1.40.0)
 Requires-Dist: packaging (<25.0)
 Requires-Dist: passlib (==1.7.4)
 Requires-Dist: prometheus-client (==0.24.1)

howler_api-4.0.0.dev799/howler/actions/add_to_bundle.py

@@ -0,0 +1,136 @@
+"""Deprecated add_to_bundle action — delegates to add_to_case via bundle_compat_service."""
+
+from typing import Optional
+
+from howler.common.exceptions import NotFoundException
+from howler.common.loader import datastore
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.services import bundle_compat_service, case_service
+from howler.utils.str_utils import sanitize_lucene_query
+
+OPERATION_ID = "add_to_bundle"
+
+
+def execute(query: str, bundle_id: Optional[str] = None, **kwargs):
+    """Add a set of hits matching the query to the specified bundle (deprecated — uses cases).
+
+    Args:
+        query (str): The query containing the matching hits
+        bundle_id (str): The ``howler.id`` of the bundle to add the hits to.
+    """
+    report = []
+
+    if not bundle_id:
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Invalid Bundle ID",
+                "message": "Bundle ID cannot be empty.",
+            }
+        ]
+
+    try:
+        case_id = bundle_compat_service.find_case_for_bundle(bundle_id)
+        if case_id is None:
+            report.append(
+                {
+                    "query": query,
+                    "outcome": "error",
+                    "title": "Invalid Bundle",
+                    "message": f"Either a hit with ID {bundle_id} does not exist, or it has no associated case.",
+                }
+            )
+            return report
+
+        ds = datastore()
+        matching_hits = ds.hit.search(query, rows=1000)["items"]
+
+        if not matching_hits:
+            report.append(
+                {
+                    "query": query,
+                    "outcome": "skipped",
+                    "title": "No Matching Hits",
+                    "message": "There were no hits matching this query.",
+                }
+            )
+            return report
+
+        added = []
+        skipped = []
+        for hit in matching_hits:
+            child_label = f"hits/{hit.howler.analytic} ({hit.howler.id})"
+            try:
+                case_service.append_case_item(
+                    case_id,
+                    item_type="hit",
+                    item_value=hit.howler.id,
+                    item_path=child_label,
+                )
+                added.append(hit.howler.id)
+            except Exception:
+                skipped.append(hit.howler.id)
+
+        if skipped:
+            report.append(
+                {
+                    "query": f"howler.id:({' OR '.join(sanitize_lucene_query(h) for h in skipped)})",
+                    "outcome": "skipped",
+                    "title": "Skipped Hits",
+                    "message": "These hits could not be added (already present or invalid).",
+                }
+            )
+
+        if added:
+            report.append(
+                {
+                    "query": f"howler.id:({' OR '.join(sanitize_lucene_query(h) for h in added)})",
+                    "outcome": "success",
+                    "title": "Executed Successfully",
+                    "message": "The specified bundle has had all matching hits added.",
+                }
+            )
+
+    except NotFoundException as e:
+        report.append(
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Failed to Execute",
+                "message": str(e),
+            }
+        )
+    except Exception as e:
+        report.append(
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Failed to Execute",
+                "message": f"Unknown exception occurred: {str(e)}",
+            }
+        )
+
+    return report
+
+
+def specification():
+    """Specify various properties of the action, such as title, descriptions, permissions and input steps."""
+    return {
+        "id": OPERATION_ID,
+        "title": "Add to Bundle (Deprecated)",
+        "priority": 6,
+        "i18nKey": f"operations.{OPERATION_ID}",
+        "description": {
+            "short": "Add a set of hits to a bundle (deprecated — uses cases)",
+            "long": execute.__doc__,
+        },
+        "roles": ["automation_basic"],
+        "steps": [
+            {
+                "args": {"bundle_id": []},
+                "options": {},
+            }
+        ],
+        "triggers": VALID_TRIGGERS,
+    }

howler_api-4.0.0.dev799/howler/actions/remove_from_bundle.py

@@ -0,0 +1,150 @@
+"""Deprecated remove_from_bundle action — delegates to case_service for item removal."""
+
+from typing import Optional
+
+from howler.common.exceptions import NotFoundException
+from howler.common.loader import datastore
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.services import bundle_compat_service, case_service
+from howler.utils.str_utils import sanitize_lucene_query
+
+OPERATION_ID = "remove_from_bundle"
+
+
+def execute(query: str, bundle_id: Optional[str] = None, **kwargs):
+    """Remove a set of hits matching the query from the specified bundle (deprecated — uses cases).
+
+    Args:
+        query (str): The query containing the matching hits
+        bundle_id (str): The ``howler.id`` of the bundle to remove the hits from.
+    """
+    report = []
+
+    if not bundle_id:
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Invalid Bundle ID",
+                "message": "Bundle ID cannot be empty.",
+            }
+        ]
+
+    try:
+        case_id = bundle_compat_service.find_case_for_bundle(bundle_id)
+        if case_id is None:
+            report.append(
+                {
+                    "query": query,
+                    "outcome": "error",
+                    "title": "Invalid Bundle",
+                    "message": f"Either a hit with ID {bundle_id} does not exist, or it has no associated case.",
+                }
+            )
+            return report
+
+        ds = datastore()
+        matching_hits = ds.hit.search(query, rows=1000)["items"]
+
+        if not matching_hits:
+            report.append(
+                {
+                    "query": query,
+                    "outcome": "skipped",
+                    "title": "No Matching Hits",
+                    "message": "There were no hits matching this query.",
+                }
+            )
+            return report
+
+        # Get the case to check which hits are actually in it
+        case = ds.case.get(case_id)
+        if case is None:
+            report.append(
+                {
+                    "query": query,
+                    "outcome": "error",
+                    "title": "Case Not Found",
+                    "message": f"Associated case {case_id} no longer exists.",
+                }
+            )
+            return report
+
+        case_item_values = {item.value for item in case.items}
+        values_to_remove = [h.howler.id for h in matching_hits if h.howler.id in case_item_values]
+        skipped_ids = [h.howler.id for h in matching_hits if h.howler.id not in case_item_values]
+
+        if skipped_ids:
+            report.append(
+                {
+                    "query": f"howler.id:({' OR '.join(sanitize_lucene_query(h) for h in skipped_ids)})",
+                    "outcome": "skipped",
+                    "title": "Skipped Hits Not in Bundle",
+                    "message": "These hits are not in the bundle.",
+                }
+            )
+
+        if not values_to_remove:
+            report.append(
+                {
+                    "query": query,
+                    "outcome": "skipped",
+                    "title": "No Matching Hits",
+                    "message": "None of the matching hits were found in the bundle.",
+                }
+            )
+            return report
+
+        case_service.remove_case_items(case_id, values_to_remove)
+
+        report.append(
+            {
+                "query": query,
+                "outcome": "success",
+                "title": "Executed Successfully",
+                "message": f"Matching hits removed from bundle with id {bundle_id}",
+            }
+        )
+
+    except NotFoundException as e:
+        report.append(
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Failed to Execute",
+                "message": str(e),
+            }
+        )
+    except Exception as e:
+        report.append(
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Failed to Execute",
+                "message": f"Unknown exception occurred: {str(e)}",
+            }
+        )
+
+    return report
+
+
+def specification():
+    """Specify various properties of the action, such as title, descriptions, permissions and input steps."""
+    return {
+        "id": OPERATION_ID,
+        "title": "Remove from Bundle (Deprecated)",
+        "priority": 5,
+        "i18nKey": f"operations.{OPERATION_ID}",
+        "description": {
+            "short": "Remove a set of hits from a bundle (deprecated — uses cases)",
+            "long": execute.__doc__,
+        },
+        "roles": ["automation_basic"],
+        "steps": [
+            {
+                "args": {"bundle_id": []},
+                "options": {},
+            }
+        ],
+        "triggers": VALID_TRIGGERS,
+    }

{howler_api-4.0.0.dev740 → howler_api-4.0.0.dev799}/howler/api/__init__.py

@@ -10,6 +10,7 @@ from howler import odm
 from howler.common.loader import APP_NAME
 from howler.common.logging import get_logger, log_with_traceback
 from howler.config import QUOTA_TRACKER, get_version
+from howler.utils.constants import TESTING
 from howler.utils.str_utils import safe_str
 
 API_PREFIX = "/api"
@@ -73,7 +74,8 @@ def _make_api_response(
             resp.set_cookie(k, v, secure=True, httponly=True, samesite="Lax")
 
     RAW_API_COUNTER.labels(request.method, str(request.url_rule), status_code).inc()
-    logger.info("%s %s - %s", request.method, request.path, status_code)
+    if not TESTING:
+        logger.info("%s %s - %s", request.method, request.path, status_code)
 
     return resp
 

{howler_api-4.0.0.dev740 → howler_api-4.0.0.dev799}/howler/api/socket.py

@@ -4,6 +4,7 @@ import os
 from typing import Any
 
 from flask import Blueprint, request
+from opentelemetry import trace
 
 import howler.services.event_service as event_service
 from howler.api import ok, unauthorized
@@ -21,10 +22,12 @@ socket_api = Blueprint("socket", "socket", url_prefix="/socket/v1")
 socket_api._doc = "Endpoints concerning websocket connectivity between the client and server"  # type: ignore
 
 logger = get_logger(__file__)
+tracer = trace.get_tracer(__name__)
 
 hit_helper = OdmHelper(Hit)
 
 
+@tracer.start_as_current_span(f"{__name__}.emit")
 @socket_api.route("/emit/<event>", methods=["POST"])
 def emit(event: str):
     """Emit an event to all listening websockets"""
@@ -46,6 +49,7 @@ def emit(event: str):
     return ok()
 
 
+@tracer.start_as_current_span(f"{__name__}.connect")
 @socket_api.route("/connect", websocket=True)  # type: ignore
 @websocket_auth(required_priv=["R"])
 def connect(ws: Server, *args: Any, ws_id: str, **kwargs):

{howler_api-4.0.0.dev740 → howler_api-4.0.0.dev799}/howler/api/v1/clue.py

@@ -1,9 +1,7 @@
-import sys
 import time
 from typing import Callable, Optional
 
 import requests
-from elasticapm.traces import capture_span
 from flask import request
 
 from howler.api import bad_gateway, make_subapi_blueprint, ok
@@ -13,6 +11,7 @@ from howler.common.swagger import generate_swagger_docs
 from howler.config import cache, config
 from howler.plugins import get_plugins
 from howler.security import api_login
+from howler.utils.constants import TESTING
 
 SUB_API = "clue"
 clue_api = make_subapi_blueprint(SUB_API, api_version=1)
@@ -23,7 +22,7 @@ logger = get_logger(__file__)
 
 def skip_cache(*args):
     "Function to skip cache in testing mode"
-    return "pytest" in sys.modules
+    return TESTING
 
 
 @cache.memoize(15 * 60, unless=skip_cache)
@@ -74,22 +73,21 @@ def proxy_to_clue(path, **kwargs):
     clue_token = get_token(auth_token)
 
     start = time.perf_counter()
-    with capture_span("clue", span_type="http"):
-        if request.method.lower() == "get":
-            response = requests.get(
-                f"{config.core.clue.url}/{path}",
-                headers={"Authorization": f"Bearer {clue_token}", "Accept": "application/json"},
-                params=request.args.to_dict(),
-                timeout=5 * 60,
-            )
-        else:
-            response = requests.post(
-                f"{config.core.clue.url}/{path}",
-                json=request.json,
-                headers={"Authorization": f"Bearer {clue_token}", "Accept": "application/json"},
-                params=request.args.to_dict(),
-                timeout=5 * 60,
-            )
+    if request.method.lower() == "get":
+        response = requests.get(
+            f"{config.core.clue.url}/{path}",
+            headers={"Authorization": f"Bearer {clue_token}", "Accept": "application/json"},
+            params=request.args.to_dict(),
+            timeout=5 * 60,
+        )
+    else:
+        response = requests.post(
+            f"{config.core.clue.url}/{path}",
+            json=request.json,
+            headers={"Authorization": f"Bearer {clue_token}", "Accept": "application/json"},
+            params=request.args.to_dict(),
+            timeout=5 * 60,
+        )
 
     logger.debug("Request to clue completed in %s ms", round(time.perf_counter() - start))
 

{howler_api-4.0.0.dev740 → howler_api-4.0.0.dev799}/howler/api/v1/hit.py

@@ -17,7 +17,7 @@ from howler.api import (
     ok,
 )
 from howler.api.v1.utils.etag import add_etag
-from howler.common.exceptions import HowlerException, HowlerValueError, InvalidDataException
+from howler.common.exceptions import HowlerException, HowlerValueError, InvalidDataException, NotFoundException
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
 from howler.common.swagger import generate_swagger_docs
@@ -976,3 +976,136 @@ def remove_react_comment(id: str, comment_id: str, user: dict[str, Any], **kwarg
     new_hit, version = hit_service.save_hit(hit, version=kwargs.get("server_version"))
 
     return ok(new_hit), version
+
+
+# ---------------------------------------------------------------------------
+# Deprecated bundle shim endpoints
+# ---------------------------------------------------------------------------
+
+
+def _deprecation_headers(response):
+    """Inject deprecation headers into a Flask Response."""
+    response.headers["Deprecation"] = "true"
+    response.headers["Sunset"] = "2027-01-01"
+    return response
+
+
+@generate_swagger_docs()
+@hit_api.route("/bundle", methods=["POST"])
+@api_login(audit=False, required_priv=["W"])
+def create_bundle(user: User, **kwargs):
+    """Create a new bundle (deprecated — creates a case instead).
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Data Block:
+    {
+        "bundle": {
+            ...hit          # A howler hit that will be used as a template for this new bundle
+        },
+        "hits": [...ids]    # A list of existing howler hits to add as children to the new bundle
+    }
+
+    Result Example:
+    {
+        ...hit      # The created bundle (synthesized from the underlying case)
+    }
+    """
+    from howler.services import bundle_compat_service
+
+    data = request.json
+    if not isinstance(data, dict):
+        return bad_request(err="Invalid data format")
+
+    bundle_hit: Optional[dict[str, Any]] = data.get("bundle")
+    if bundle_hit is None:
+        return bad_request(err="You did not provide a bundle hit.")
+
+    child_hits: list[str] = data.get("hits", [])
+
+    try:
+        result = bundle_compat_service.create_bundle(bundle_hit, child_hits, user=user.uname)
+        return _deprecation_headers(created(result))
+    except HowlerException as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@hit_api.route("/bundle/<id>", methods=["PUT"])
+@api_login(audit=False, required_priv=["W"])
+def update_bundle(id, **kwargs):
+    """Add hits to a bundle (deprecated — adds items to the underlying case).
+
+    Variables:
+    id  => The ID of the bundle to update
+
+    Arguments:
+    None
+
+    Data Block:
+    [
+        ...ids
+    ]
+
+    Result Example:
+    {
+        ...hit      # The updated bundle (synthesized from the underlying case)
+    }
+    """
+    from howler.services import bundle_compat_service
+    from howler.services.bundle_compat_service import BundleConflictException
+
+    hit_ids = request.json
+    if not isinstance(hit_ids, list):
+        return bad_request(err="Invalid data format")
+
+    try:
+        result = bundle_compat_service.add_to_bundle(id, hit_ids)
+        return _deprecation_headers(ok(result))
+    except BundleConflictException as e:
+        return conflict(err=str(e))
+    except NotFoundException as e:
+        return not_found(err=str(e))
+    except HowlerException as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@hit_api.route("/bundle/<id>", methods=["DELETE"])
+@api_login(audit=False, required_priv=["W"])
+def remove_bundle_children(id, **kwargs):
+    """Remove hits from a bundle (deprecated — removes items from the underlying case).
+
+    Variables:
+    id  => The ID of the bundle to update
+
+    Arguments:
+    None
+
+    Data Block:
+    [
+        ...ids OR '*'   # A list of ids to remove, or a single '*' to remove all
+    ]
+
+    Result Example:
+    {
+        ...hit      # The updated hit (synthesized from the underlying case)
+    }
+    """
+    from howler.services import bundle_compat_service
+
+    hit_ids = request.json
+    if not isinstance(hit_ids, list):
+        return bad_request(err="Invalid data format")
+
+    try:
+        result = bundle_compat_service.remove_from_bundle(id, hit_ids)
+        return _deprecation_headers(ok(result))
+    except NotFoundException as e:
+        return not_found(err=str(e))
+    except HowlerException as e:
+        return bad_request(err=str(e))

{howler_api-4.0.0.dev740 → howler_api-4.0.0.dev799}/howler/api/v1/search.py

@@ -556,11 +556,13 @@ def count(index, **kwargs):
         params.update({"access_control": user["access_control"]})
 
     query = req_data.get("query", None)
+    filters = req_data.get("filters", [])
+
     if not query:
         return bad_request(err="There was no search query.")
 
     try:
-        return ok(collection().count(query, **params))
+        return ok(collection().count(query, filters=filters, **params))
     except (SearchException, BadRequestError) as e:
         return bad_request(err=f"SearchException: {e}")
 

{howler_api-4.0.0.dev740 → howler_api-4.0.0.dev799}/howler/api/v1/tool.py

@@ -77,8 +77,11 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
     warnings = []
     # Validate field_map targets
     hit_fields = Hit.flat_fields()
+    _bundle_compat_fields = {"howler.is_bundle", "howler.hits", "howler.bundle_size", "howler.bundles"}
     for targets in field_map.values():
         for target in targets:
+            if target in _bundle_compat_fields:
+                continue
             # This is checking to see if the target matches one of two cases:
             # Simple fields - hit.obj.key of type str (should match)
             # Compound fields - hit.obj of type dict (should also match)
@@ -93,7 +96,9 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
                     return bad_request(err=warning)
 
     out: list[dict[str, Any]] = []
-    odms = []
+    odms: list[Hit] = []
+    bundle_raw: dict[str, Any] | None = None
+    bundle_index: int | None = None
     for hit in hits:
         cur_id = get_random_id()
         cur_time = now_as_iso()
@@ -135,8 +140,19 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
                         obj[target] = _val
 
         try:
+            is_bundle = obj.pop("howler.is_bundle", False)
+            obj.pop("howler.hits", None)
+            obj.pop("howler.bundle_size", None)
+            obj.pop("howler.bundles", None)
+
             odm, warns = hit_service.convert_hit(obj, unique=True, ignore_extra_values=ignore_extra_values)
 
+            if is_bundle:
+                if bundle_raw is not None:
+                    return bad_request(err="You can only specify one bundle hit!")
+                bundle_raw = obj
+                bundle_index = len(odms)
+
             odms.append(odm)
 
             out.append(
@@ -155,12 +171,36 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
     if any([obj["error"] for obj in out]):
         return bad_request(out, warnings=warnings, err="No valid hits were provided")
     else:
-        for odm in odms:
-            hit_service.create_hit(odm.howler.id, odm, user=user.uname)
-            analytic_service.save_from_hit(odm, user)
+        if bundle_index is not None:
+            # Route through bundle compat service → creates a case
+            from howler.services import bundle_compat_service
+
+            bundle_odm = odms[bundle_index]
+            child_odms = [odm for i, odm in enumerate(odms) if i != bundle_index]
+
+            for odm in child_odms:
+                hit_service.create_hit(odm.howler.id, odm, user=user.uname)
+                analytic_service.save_from_hit(odm, user)
+
+            child_ids = [odm.howler.id for odm in child_odms]
+            bundle_data = bundle_odm.as_primitives()
+            result = bundle_compat_service.create_bundle(bundle_data, child_ids, user=user.uname)
+            warnings.append(bundle_compat_service.DEPRECATION_MESSAGE)
+
+            # Replace the bundle entry in the output with the created bundle id
+            for entry in out:
+                if entry.get("id") == bundle_odm.howler.id:
+                    entry["id"] = result["howler"]["id"]
+                    entry["_case_id"] = result.get("_case_id")
+        else:
+            for odm in odms:
+                hit_service.create_hit(odm.howler.id, odm, user=user.uname)
+                analytic_service.save_from_hit(odm, user)
 
         datastore().hit.commit()
 
-        action_service.bulk_execute_on_query(f"howler.id:({' OR '.join(entry['id'] for entry in out)})", user=user)
+        action_service.bulk_execute_on_query(
+            f"howler.id:({' OR '.join(entry['id'] for entry in out if entry['id'])})", user=user
+        )
 
         return created(out, warnings=warnings)