howler-api 2.10.0.dev51__tar.gz

howler_api-2.10.0.dev51/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.3
+Name: howler-api
+Version: 2.10.0.dev51
+Summary: Howler - API server
+License: MIT
+Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
+Author: Canadian Centre for Cyber Security
+Author-email: howler@cyber.gc.ca
+Maintainer: Matthew Rafuse
+Maintainer-email: matthew.rafuse@cyber.gc.ca
+Requires-Python: >=3.9.17,<4.0.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: apscheduler (==3.10.4)
+Requires-Dist: authlib (>=1.6.0,<2.0.0)
+Requires-Dist: azure-identity (==1.16.1)
+Requires-Dist: azure-storage-blob (==12.14.1)
+Requires-Dist: chardet (==5.1.0)
+Requires-Dist: chevron (==0.14.0)
+Requires-Dist: elastic-apm[flask] (>=6.22.0,<7.0.0)
+Requires-Dist: elasticsearch (==8.6.1)
+Requires-Dist: flasgger (>=0.9.7.1,<0.10.0.0)
+Requires-Dist: flask (==2.2.5)
+Requires-Dist: flask-caching (==2.0.2)
+Requires-Dist: gevent (==23.9.1)
+Requires-Dist: gunicorn (==23.0.0)
+Requires-Dist: luqum (>=1.0.0,<2.0.0)
+Requires-Dist: mergedeep (>=1.3.4,<2.0.0)
+Requires-Dist: netifaces (==0.11.0)
+Requires-Dist: packaging (<25.0)
+Requires-Dist: passlib (==1.7.4)
+Requires-Dist: prometheus-client (==0.17.1)
+Requires-Dist: pydantic (>=2.11.4,<3.0.0)
+Requires-Dist: pydantic-settings[yaml] (>=2.9.1,<3.0.0)
+Requires-Dist: pydash (>=8.0.5,<9.0.0)
+Requires-Dist: pyjwt (==2.6.0)
+Requires-Dist: pyroute2-core (==0.6.13)
+Requires-Dist: pysftp (==0.2.9)
+Requires-Dist: pysigma (==0.11.17)
+Requires-Dist: pysigma-backend-elasticsearch (>=1.1.2,<2.0.0)
+Requires-Dist: python-baseconv (==1.2.2)
+Requires-Dist: python-datemath (==3.0.3)
+Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
+Requires-Dist: pyyaml (==6.0.2)
+Requires-Dist: redis (==4.5.4)
+Requires-Dist: requests (==2.32.2)
+Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
+Requires-Dist: validators (>=0.34.0,<0.35.0)
+Requires-Dist: wsproto (==1.2.0)
+Project-URL: Documentation, https://cybercentrecanada.github.io/howler-docs/developer/backend/
+Project-URL: Homepage, https://cybercentrecanada.github.io/howler-docs/
+Project-URL: Repository, https://github.com/CybercentreCanada/howler-api
+Description-Content-Type: text/markdown
+
+# Howler API
+
+## Introduction
+
+Howler is an application that allows analysts to triage hits and alerts. It provides a way for analysts to efficiently review and analyze alerts generated by different analytics and detections.
+
+## Contributing
+
+See [CONTRIBUTING.md](doc/CONTRIBUTING.md).
+

howler_api-2.10.0.dev51/README.md

@@ -0,0 +1,9 @@
+# Howler API
+
+## Introduction
+
+Howler is an application that allows analysts to triage hits and alerts. It provides a way for analysts to efficiently review and analyze alerts generated by different analytics and detections.
+
+## Contributing
+
+See [CONTRIBUTING.md](doc/CONTRIBUTING.md).

howler_api-2.10.0.dev51/howler/actions/__init__.py

@@ -0,0 +1,154 @@
+import importlib
+import os
+import re
+from pathlib import Path
+from typing import Any, Optional
+
+from howler.common.logging import get_logger
+from howler.config import config
+from howler.odm.models.user import User
+
+logger = get_logger(__file__)
+
+PLUGIN_PATH = Path(os.environ.get("HWL_PLUGIN_DIRECTORY", "/etc/howler/plugins"))
+
+
+def __sanitize_specification(spec: dict[str, Any]) -> dict[str, Any]:
+    """Adapt the specification for use in the UI
+
+    Args:
+        spec (dict[str, Any]): The raw specification
+
+    Returns:
+        dict[str, Any]: The sanitized specification for use in the UI
+    """
+    return {
+        **spec,
+        "description": {
+            **spec["description"],
+            "long": re.sub(r"\n +(request_id|query).+", "", spec["description"]["long"])
+            .replace("\n    ", "\n")
+            .replace("Args:", "Args:\n"),
+        },
+        "steps": [{**step, "args": {k: list(v) for k, v in step["args"].items()}} for step in spec["steps"]],
+    }
+
+
+def __sanitize_report(report: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Deduplicate identical entries with different queries
+
+    Args:
+        report (list[dict[str, Any]]): The unsanitized, verbose report
+
+    Returns:
+        list[dict[str, Any]]: The sanitized, concise report
+    """
+    by_message: dict[str, Any] = {}
+
+    for entry in report:
+        # if these three keys match, we should merge the queries that use them both. For example, when multiple hits
+        # fail to transition for the same reason.
+        key = f"{entry['title']}==={entry['message']}==={entry['outcome']}"
+
+        if key in by_message:
+            by_message[key].append(f'({entry["query"]})')
+        else:
+            by_message[key] = [f'({entry["query"]})']
+
+    sanitized: list[dict[str, Any]] = []
+    for key, queries in by_message.items():
+        (title, message, outcome) = key.split("===")
+
+        sanitized.append(
+            {
+                "query": " OR ".join(queries),
+                "outcome": outcome,
+                "title": title,
+                "message": message,
+            }
+        )
+
+    return sanitized
+
+
+def execute(
+    operation_id: str,
+    query: str,
+    user: User,
+    request_id: Optional[str] = None,
+    **kwargs,
+) -> list[dict[str, Any]]:
+    """Execute a specification
+
+    Args:
+        operation_id (str): The id of the operation to run
+        query (str): The query to run this action on
+        user (dict[str, Any]): The user running this action
+        request_id (str, None): A user-provided ID, can be used to track the progress of their excecution via websockets
+
+    Returns:
+        list[dict[str, Any]]: A report on the execution
+    """
+    try:
+        automation = importlib.import_module(f"howler.actions.{operation_id}")
+    except Exception as e:
+        logger.critical("Error when importing %s - %s", operation_id, e)
+
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Unknown Action",
+                "message": f"The operation ID provided ({operation_id}) does not match any enabled operations.",
+            }
+        ]
+
+    missing_roles = set(automation.specification()["roles"]) - set(user["type"])
+    if missing_roles:
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Insufficient permissions",
+                "message": (
+                    f"The operation ID provided ({operation_id}) requires permissions you do not have "
+                    f"({', '.join(missing_roles)}). Contact HOWLER Support for more information."
+                ),
+            }
+        ]
+
+    report = automation.execute(query=query, request_id=request_id, user=user, **kwargs)
+
+    return __sanitize_report(report)
+
+
+def specifications() -> list[dict[str, Any]]:
+    """A list of specifications for the available operations
+
+    Returns:
+        list[dict[str, Any]]: A list of specifications
+    """
+    specifications = []
+
+    module_paths = {"howler": Path(__file__).parent}
+
+    for plugin in config.core.plugins:
+        plugin_module_path = PLUGIN_PATH / plugin / "actions"
+        if plugin_module_path.exists():
+            module_paths[plugin] = plugin_module_path
+
+    for module_name, module_path in module_paths.items():
+        for module in (
+            _file
+            for _file in module_path.iterdir()
+            if _file.suffix == ".py" and _file.name not in ["__init__.py", "example_plugin.py"]
+        ):
+            try:
+                automation = importlib.import_module(f"{module_name}.actions.{module.stem}")
+
+                specifications.append(__sanitize_specification(automation.specification()))
+
+            except Exception:  # pragma: no cover
+                logger.exception("Error when initializing %s", module)
+
+    return specifications

howler_api-2.10.0.dev51/howler/actions/add_label.py

@@ -0,0 +1,111 @@
+from typing import Optional
+
+from howler.common.loader import datastore
+from howler.datastore.operations import OdmHelper
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.odm.models.hit import Hit
+from howler.odm.models.howler_data import Label
+from howler.utils.str_utils import sanitize_lucene_query
+
+hit_helper = OdmHelper(Hit)
+
+OPERATION_ID = "add_label"
+
+CATEGORIES = list(Label.fields().keys())
+
+
+def execute(query: str, category: str = "generic", label: Optional[str] = None, **kwargs):
+    """Add a label to a hit.
+
+    Args:
+        query (str): The query on which to apply this automation.
+        category (str, optional): The category of label to add. Defaults to "generic".
+        label (str): The label content. Defaults to None.
+    """
+    if category not in CATEGORIES:
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Invalid Category",
+                "message": f"'{category}' is not a valid category.",
+            }
+        ]
+
+    if not label:
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Invalid Label",
+                "message": "Label cannot be empty.",
+            }
+        ]
+
+    report = []
+
+    ds = datastore()
+
+    skipped_hits = ds.hit.search(
+        f"({query}) AND howler.labels.{category}:{sanitize_lucene_query((label))}",
+        fl="howler.id",
+    )["items"]
+
+    if len(skipped_hits) > 0:
+        report.append(
+            {
+                "query": f"howler.id:({' OR '.join(h.howler.id for h in skipped_hits)})",
+                "outcome": "skipped",
+                "title": "Skipped Hit with Label",
+                "message": f"These hits already have the label {label}.",
+            }
+        )
+
+    try:
+        ds.hit.update_by_query(
+            query,
+            [hit_helper.list_add(f"howler.labels.{category}", label, if_missing=True)],
+        )
+
+        report.append(
+            {
+                "query": query,
+                "outcome": "success",
+                "title": "Executed Successfully",
+                "message": f"Label '{label}' added to category '{category}' for all matching hits.",
+            }
+        )
+    except Exception as e:
+        report.append(
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Failed to Execute",
+                "message": f"Unknown exception occurred: {str(e)}",
+            }
+        )
+
+    return report
+
+
+def specification():
+    """Specify various properties of the action, such as title, descriptions, permissions and input steps."""
+    return {
+        "id": OPERATION_ID,
+        "title": "Add Label",
+        "priority": 8,
+        "i18nKey": "operations.add_label",
+        "description": {
+            "short": "Add a label to a hit",
+            "long": execute.__doc__,
+        },
+        "roles": ["automation_basic"],
+        "steps": [
+            {
+                "args": {"category": [], "label": []},
+                "options": {"category": CATEGORIES},
+                "validation": {"warn": {"query": "howler.labels.$category:$label"}},
+            }
+        ],
+        "triggers": VALID_TRIGGERS,
+    }

howler_api-2.10.0.dev51/howler/actions/add_to_bundle.py

@@ -0,0 +1,159 @@
+from typing import Optional
+
+from howler.common.loader import datastore
+from howler.datastore.operations import OdmHelper
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.odm.models.hit import Hit
+from howler.services import hit_service
+from howler.utils.str_utils import sanitize_lucene_query
+
+hit_helper = OdmHelper(Hit)
+
+OPERATION_ID = "add_to_bundle"
+
+
+def execute(query: str, bundle_id: Optional[str] = None, **kwargs):
+    """Add a set of hits matching the query to the specified bundle.
+
+    Args:
+        query (str): The query containing the matching hits
+        bundle_id (str): The `howler.id` of the bundle to add the hits to.
+    """
+    report = []
+
+    if not bundle_id:
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Invalid Bundle ID",
+                "message": "Bundle ID cannot be empty.",
+            }
+        ]
+
+    try:
+        bundle_hit = hit_service.get_hit(bundle_id, as_odm=True)
+        if not bundle_hit or not bundle_hit.howler.is_bundle:
+            report.append(
+                {
+                    "query": query,
+                    "outcome": "error",
+                    "title": "Invalid Bundle",
+                    "message": f"Either a hit with ID {bundle_id} does not exist, or it is not a bundle.",
+                }
+            )
+            return report
+
+        ds = datastore()
+
+        skipped_hits_bundles = ds.hit.search(
+            f"({query}) AND howler.is_bundle:true",
+            fl="howler.id",
+        )["items"]
+
+        if len(skipped_hits_bundles) > 0:
+            report.append(
+                {
+                    "query": f"({query}) AND howler.is_bundle:true",
+                    "outcome": "skipped",
+                    "title": "Skipped Bundles",
+                    "message": "Bundles cannot be added to a bundle.",
+                }
+            )
+
+        skipped_hits_already_added = ds.hit.search(
+            f"({query}) AND (howler.bundles:{sanitize_lucene_query(bundle_id)})",
+            fl="howler.id",
+        )["items"]
+
+        if len(skipped_hits_already_added) > 0:
+            report.append(
+                {
+                    "query": f"({query}) AND (howler.bundles:{sanitize_lucene_query(bundle_id)})",
+                    "outcome": "skipped",
+                    "title": "Skipped Hits",
+                    "message": "These hits have already been added to the specified bundle.",
+                }
+            )
+
+        safe_query = f"({query}) AND (-howler.bundles:({sanitize_lucene_query(bundle_id)}) AND howler.is_bundle:false)"
+        matching_hits = ds.hit.search(safe_query)["items"]
+        if len(matching_hits) < 1:
+            report.append(
+                {
+                    "query": safe_query,
+                    "outcome": "skipped",
+                    "title": "No Matching Hits",
+                    "message": "There were no hits matching this query.",
+                }
+            )
+            return report
+
+        ds.hit.update_by_query(
+            safe_query,
+            [hit_helper.list_add("howler.bundles", sanitize_lucene_query(bundle_id), if_missing=True)],
+        )
+
+        operations = [
+            hit_helper.list_add(
+                "howler.hits",
+                hit["howler"]["id"],
+                if_missing=True,
+            )
+            for hit in matching_hits
+        ]
+
+        operations.append(hit_helper.update("howler.bundle_size", len(operations)))
+        hit_service.update_hit(
+            bundle_id,
+            operations,
+        )
+        bundle_hit = hit_service.get_hit(bundle_id, as_odm=True)
+        report.append(
+            {
+                "query": safe_query.replace("-howler.bundles", "howler.bundles"),
+                "outcome": "success",
+                "title": "Executed Successfully",
+                "message": "The specified bundle has had all matching hits added.",
+            }
+        )
+    except Exception as e:
+        report.append(
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Failed to Execute",
+                "message": f"Unknown exception occurred: {str(e)}",
+            }
+        )
+
+    return report
+
+
+def specification():
+    """Specify various properties of the action, such as title, descriptions, permissions and input steps."""
+    return {
+        "id": OPERATION_ID,
+        "title": "Add to Bundle",
+        "priority": 6,
+        "i18nKey": f"operations.{OPERATION_ID}",
+        "description": {
+            "short": "Add a set of hits to a bundle",
+            "long": execute.__doc__,
+        },
+        "roles": ["automation_basic"],
+        "steps": [
+            {
+                "args": {"bundle_id": []},
+                "options": {},
+                "validation": {
+                    "warn": {"query": "howler.bundles:($bundle_id) OR howler.is_bundle:true"},
+                    "error": {
+                        "query": "howler.id:$bundle_id AND howler.is_bundle:false",
+                        "message": "The bundle id given must be a bundle.",
+                    },
+                },
+            }
+        ],
+        "triggers": VALID_TRIGGERS,
+    }

howler_api-2.10.0.dev51/howler/actions/change_field.py

@@ -0,0 +1,76 @@
+from howler.common.loader import datastore
+from howler.datastore.operations import OdmHelper
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.odm.models.hit import Hit
+
+hit_helper = OdmHelper(Hit)
+
+OPERATION_ID = "change_field"
+
+
+def execute(query: str, field: str, value: str, **kwargs):
+    """Change one of the fields of a hit
+
+    Args:
+        query (str): The query to run this action on
+        field (str): The field to update.
+        value (str): The value to set it to. Must be a string.
+    """
+    if field not in Hit.flat_fields():
+        return [
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Invalid field",
+                "message": (f"Field '{field}' does not exist. You must pick a valid entry from the howler index."),
+            }
+        ]
+
+    report = []
+
+    try:
+        datastore().hit.update_by_query(
+            query,
+            [hit_helper.update(field, value)],
+        )
+
+        report.append(
+            {
+                "query": query,
+                "outcome": "success",
+                "title": "Executed Successfully",
+                "message": f"Field '{field}' updated to value '{value}' for all matching hits.",
+            }
+        )
+    except Exception as e:
+        report.append(
+            {
+                "query": query,
+                "outcome": "error",
+                "title": "Failed to Execute",
+                "message": str(e),
+            }
+        )
+
+    return report
+
+
+def specification():
+    """Specify various properties of the action, such as title, descriptions, permissions and input steps."""
+    return {
+        "id": OPERATION_ID,
+        "title": "Change Field",
+        "i18nKey": f"operations.{OPERATION_ID}",
+        "description": {
+            "short": "Change one of the fields of a hit",
+            "long": execute.__doc__,
+        },
+        "roles": ["automation_advanced", "admin"],
+        "steps": [
+            {
+                "args": {"field": [], "value": []},
+                "options": {"field": list(Hit.flat_fields().keys())},
+            }
+        ],
+        "triggers": VALID_TRIGGERS,
+    }