howler-api 3.0.0.dev374__py3-none-any.whl

howler/api/v1/template.py

@@ -0,0 +1,206 @@
+from flask import request
+
+from howler.api import (
+    bad_request,
+    conflict,
+    created,
+    forbidden,
+    make_subapi_blueprint,
+    no_content,
+    not_found,
+    ok,
+)
+from howler.common.exceptions import HowlerException
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.common.swagger import generate_swagger_docs
+from howler.datastore.operations import OdmHelper
+from howler.odm.models.template import Template
+from howler.odm.models.user import User
+from howler.security import api_login
+from howler.utils.str_utils import sanitize_lucene_query
+
+SUB_API = "template"
+template_api = make_subapi_blueprint(SUB_API, api_version=1)
+template_api._doc = "Manage the different templates created for viewing hits"
+
+logger = get_logger(__file__)
+
+template_helper = OdmHelper(Template)
+
+
+@generate_swagger_docs()
+@template_api.route("/", methods=["GET"])
+@api_login(required_priv=["R"])
+def get_templates(**kwargs):
+    """Get a list of templates the user can use to render hits
+
+    Variables:
+    None
+
+    Optional Arguments:
+    None
+
+    Result Example:
+    [
+        ...templates    # A list of templates the user can use
+    ]
+    """
+    try:
+        return ok(
+            datastore().template.search(
+                f"type:global OR owner:{kwargs['user']['uname']}",
+                as_obj=False,
+                rows=10000,
+            )["items"]
+        )
+    except ValueError as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@template_api.route("/", methods=["POST"])
+@api_login(required_priv=["R", "W"])
+def create_template(**kwargs):
+    """Create a new template
+
+    Variables:
+    None
+
+    Optional Arguments:
+    None
+
+    Data Block:
+    {
+        "analytic": "analytic name"             # The analytic this template applies to
+        "detection": "detection name"           # The detection this template applies to
+        "type": "global"                        # The type of template to create
+        "keys": ["howler.id", "howler.hash"]    # The keys to show when this template matches a hit
+    }
+
+    Result Example:
+    {
+        ...template            # The new template data
+    }
+    """
+    template_data = request.json
+    if not isinstance(template_data, dict):
+        return bad_request(err="Invalid data format")
+
+    if "keys" not in template_data:
+        return bad_request(err="You must specify a list of keys when creating a template!")
+
+    storage = datastore()
+
+    try:
+        template = Template(template_data)
+
+        if template.type == "personal":
+            template.owner = kwargs["user"]["uname"]
+        else:
+            template.owner = None
+
+        query_str = f"analytic:{sanitize_lucene_query(template.analytic)} AND type:{template.type}"
+
+        if template.type == "personal":
+            query_str += f" AND owner:{template.owner}"
+
+        if template.detection:
+            query_str += f" AND detection:{template.detection}"
+        else:
+            query_str += " AND -_exists_:detection"
+
+        if storage.template.search(query_str)["total"] > 0:
+            return conflict(err="A template covering this case already exists.")
+
+        storage.template.save(template.template_id, template)
+        return created(template)
+    except HowlerException as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@template_api.route("/<id>", methods=["DELETE"])
+@api_login(required_priv=["W"])
+def delete_template(id: str, user: User, **kwargs):
+    """Delete a template
+
+    Variables:
+    id => The id of the template to delete
+
+    Optional Arguments:
+    None
+
+    Data Block:
+    None
+
+    Result Example:
+    {
+        "success": true     # Did the deletion succeed?
+    }
+    """
+    storage = datastore()
+
+    if not storage.template.exists(id):
+        return not_found(err="This template does not exist")
+
+    existing_template: Template = storage.template.get_if_exists(id)
+
+    if existing_template.type == "personal" and existing_template.owner != user.uname:
+        return forbidden(err="You cannot delete a personal template that is not owned by you.")
+
+    if existing_template.type == "global" and "admin" not in user.type:
+        return forbidden(err="You cannot delete a global template unless you are an administrator.")
+
+    result = storage.template.delete(id)
+    if result:
+        return no_content()
+    else:
+        return not_found()
+
+
+@generate_swagger_docs()
+@template_api.route("/<id>", methods=["PUT"])
+@api_login(required_priv=["R", "W"])
+def update_template_fields(id: str, user: User, **kwargs):
+    """Update a template's keys
+
+    Variables:
+    id => The id of the template to modify
+
+    Optional Arguments:
+    None
+
+    Data Block:
+    [
+        "howler.id",
+        "howler.hash"
+    ]
+
+    Result Example:
+    {
+        ...template     # The updated template data
+    }
+    """
+    storage = datastore()
+
+    if not storage.template.exists(id):
+        return not_found(err="This template does not exist")
+
+    new_fields = request.json
+    if not isinstance(new_fields, list) or not all(isinstance(f, str) for f in new_fields):
+        return bad_request(err="List of new fields must be a list of strings.")
+
+    existing_template: Template = storage.template.get_if_exists(id)
+
+    if existing_template.type == "personal" and existing_template.owner != user.uname:
+        return forbidden(err="You cannot update a personal template that is not owned by you.")
+
+    existing_template.keys = new_fields
+
+    storage.template.save(existing_template.template_id, existing_template)
+
+    try:
+        return ok(storage.template.get_if_exists(existing_template.template_id, as_obj=False))
+    except HowlerException as e:
+        return bad_request(err=str(e))

howler/api/v1/tool.py

@@ -0,0 +1,183 @@
+from typing import Any, Optional
+
+from flask import request
+
+from howler.api import bad_request, created, make_subapi_blueprint
+from howler.common.exceptions import HowlerException
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.common.swagger import generate_swagger_docs
+from howler.datastore.operations import OdmHelper
+from howler.odm.base import _Field
+from howler.odm.models.hit import Hit
+from howler.odm.models.user import User
+from howler.security import api_login
+from howler.services import action_service, analytic_service, hit_service
+from howler.utils.dict_utils import flatten
+from howler.utils.isotime import now_as_iso
+from howler.utils.str_utils import get_parent_key
+from howler.utils.uid import get_random_id
+
+SUB_API = "tools"
+tool_api = make_subapi_blueprint(SUB_API, api_version=1)
+tool_api._doc = "Manage the tools"
+
+logger = get_logger(__file__)
+
+hit_helper = OdmHelper(Hit)
+
+
+@generate_swagger_docs()
+@tool_api.route("/<tool_name>/hits", methods=["POST", "PUT"])
+@api_login(required_priv=["W"])
+def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
+    """Create one or many hits for a tool using field mapping.
+
+    Variables:
+    tool_name   => Name of the tool the hit is for
+
+    Arguments:
+    None
+
+    Data Block:
+    {
+        "map": {            # For each field in the hit, list of field data will be copied to
+            "source.field.in.raw.data": ["target.field.in.howler.hit.index"],
+            ...
+        },
+        "hits": [           # List of raw hits to create the hit from
+            {...},
+            {...}
+        ]
+    }
+
+    Result Example:
+    {
+        [                   # List of hits IDs/Errors created of the different hits (preserved order)
+            {'id': "id1", 'error': None},
+            {'id': "id2", 'error': None},
+            {'id': None, 'error': "Error message"},
+        ]
+    }
+    """
+    data = request.json
+    if not isinstance(data, dict):
+        return bad_request(err="Invalid data format")
+
+    field_map = data.pop("map", None)
+    hits = data.pop("hits", None)
+    ignore_extra_values: bool = bool(request.args.get("ignore_extra_values", False, type=lambda v: v.lower() == "true"))
+    logger.debug(f"ignore_extra_values = {ignore_extra_values}")
+    # Check data type
+    if not isinstance(field_map, dict):
+        return bad_request(err="Invalid: 'map' field is missing or invalid.")
+
+    if not isinstance(hits, list):
+        return bad_request(err="Invalid: 'hits' field is missing or invalid.")
+    warnings = []
+    # Validate field_map targets
+    hit_fields = Hit.flat_fields()
+    for targets in field_map.values():
+        for target in targets:
+            # This is checking to see if the target matches one of two cases:
+            # Simple fields - hit.obj.key of type str (should match)
+            # Compound fields - hit.obj of type dict (should also match)
+            # This allows significantly easier creation of hits, since you don't need to deconstruct every dict into
+            # individual fields
+            if target not in hit_fields and not any(f for f in hit_fields.keys() if get_parent_key(f) == target):
+                warning = f"Invalid target field in the map: {target}"
+                if ignore_extra_values:
+                    warnings.append(warning)
+                    # field_map.pop(target)
+                else:
+                    return bad_request(err=warning)
+
+    out: list[dict[str, Any]] = []
+    odms = []
+    bundle_hit: Optional[Hit] = None
+    for hit in hits:
+        cur_id = get_random_id()
+        cur_time = now_as_iso()
+        obj: dict[str, Any] = {
+            "agent.type": tool_name,
+            "event.created": cur_time,
+            "event.id": cur_id,
+            "howler.id": cur_id,
+            "howler.analytic": tool_name,
+            "howler.score": 0,
+        }
+        hit = flatten(hit)
+        for source, targets in field_map.items():
+            val = hit.get(source, None)
+            if val is not None:
+                for target in targets:
+                    _val = val
+                    try:
+                        field_data: Optional[_Field] = hit_fields[target]
+                    except KeyError:
+                        logger.debug(f"`{target}` not in hit fields")
+                        field_data = next(
+                            (v for k, v in hit_fields.items() if get_parent_key(k) == target),
+                            None,
+                        )
+
+                    if field_data is not None and field_data.multivalued:
+                        if not isinstance(_val, list):
+                            _val = [val]
+                        obj.setdefault(target, [])
+                        obj[target].extend(_val)
+                    else:
+                        if isinstance(val, list):
+                            if not len(val):
+                                continue
+
+                            _val = val[0]
+
+                        obj[target] = _val
+
+        try:
+            odm, warns = hit_service.convert_hit(obj, unique=True, ignore_extra_values=ignore_extra_values)
+
+            if odm.howler.is_bundle and bundle_hit is None:
+                bundle_hit = odm
+            elif odm.howler.is_bundle:
+                return bad_request(err="You can only specify one bundle hit!")
+            else:
+                odms.append(odm)
+
+            out.append(
+                {
+                    "id": odm.howler.id,
+                    "error": None,
+                    "warn": warns,
+                }
+            )
+        except HowlerException as e:
+            logger.warning(f"{type(e).__name__} when saving {cur_id}!")
+            logger.warning(e)
+
+            out.append({"id": None, "error": str(e)})
+    # If there are any errors...
+    if any([obj["error"] for obj in out]):
+        return bad_request(out, warnings=warnings, err="No valid hits were provided")
+    else:
+        for odm in odms:
+            if bundle_hit is not None:
+                bundle_hit.howler.hits.append(odm.howler.id)
+                bundle_hit.howler.bundle_size += 1
+                odm.howler.bundles.append(bundle_hit.howler.id)
+
+            hit_service.create_hit(odm.howler.id, odm, user=user["uname"])
+
+            analytic_service.save_from_hit(odm, user)
+
+        if bundle_hit:
+            hit_service.create_hit(bundle_hit.howler.id, bundle_hit, user=user["uname"])
+
+            analytic_service.save_from_hit(bundle_hit, user)
+
+        datastore().hit.commit()
+
+        action_service.bulk_execute_on_query(f"howler.id:({' OR '.join(entry['id'] for entry in out)})", user=user)
+
+        return created(out, warnings=warnings)