howler-api 2.13.0.dev329__py3-none-any.whl

howler/api/v1/view.py

@@ -0,0 +1,288 @@
+from flask import request
+from mergedeep.mergedeep import merge
+
+from howler.api import bad_request, created, forbidden, make_subapi_blueprint, no_content, not_found, ok
+from howler.common.exceptions import HowlerException
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.common.swagger import generate_swagger_docs
+from howler.datastore.exceptions import SearchException
+from howler.odm.models.user import User
+from howler.odm.models.view import View
+from howler.security import api_login
+
+SUB_API = "view"
+view_api = make_subapi_blueprint(SUB_API, api_version=1)
+view_api._doc = "Manage the different views created for filtering hits"
+
+logger = get_logger(__file__)
+
+
+@generate_swagger_docs()
+@view_api.route("/", methods=["GET"])
+@api_login(required_priv=["R"])
+def get_views(user: User, **kwargs):
+    """Get a list of views the user can use to filter hits
+
+    Variables:
+    None
+
+    Optional Arguments:
+    None
+
+    Result Example:
+    [
+        ...views    # A list of views the user can use
+    ]
+    """
+    try:
+        return ok(
+            datastore().view.search(
+                f"type:global OR owner:({user['uname']} OR none)", as_obj=False, rows=1000, sort="title asc"
+            )["items"]
+        )
+    except ValueError as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@view_api.route("/", methods=["POST"])
+@api_login(required_priv=["R", "W"])
+def create_view(**kwargs):
+    """Create a new view
+
+    Variables:
+    None
+
+    Optional Arguments:
+    None
+
+    Data Block:
+    {
+        "title": "New View"     # The name of this view
+        "query": "howler.id:*"  # The query to run
+        "type": "global"        # The type of view - personal or global
+    }
+
+    Result Example:
+    {
+        ...view            # The new view data
+    }
+    """
+    view_data = request.json
+    if not isinstance(view_data, dict):
+        return bad_request(err="Invalid data format")
+
+    if "title" not in view_data:
+        return bad_request(err="You must specify a title when creating a view.")
+
+    if "query" not in view_data:
+        return bad_request(err="You must specify a query when creating a view.")
+
+    if "type" not in view_data:
+        return bad_request(err="You must specify a type when creating a view.")
+
+    storage = datastore()
+
+    try:
+        # Make sure the query is valid
+        storage.hit.search(view_data["query"])
+
+        view = View(view_data)
+
+        view.owner = kwargs["user"]["uname"]
+
+        if view.type == "personal":
+            current_user = storage.user.get_if_exists(kwargs["user"]["uname"])
+
+            current_user["favourite_views"] = current_user.get("favourite_views", []) + [view.view_id]
+
+            storage.user.save(current_user["uname"], current_user)
+
+        storage.view.save(view.view_id, view)
+        return created(view)
+    except SearchException:
+        return bad_request(err="You must use a valid query when creating a view.")
+    except HowlerException as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@view_api.route("/<view_id>", methods=["DELETE"])
+@api_login(required_priv=["W"])
+def delete_view(view_id: str, user: User, **kwargs):
+    """Delete a view
+
+    Variables:
+    view_id => The id of the view to delete
+
+    Optional Arguments:
+    None
+
+    Data Block:
+    None
+
+    Result Example:
+    {
+        "success": true     # Did the deletion succeed?
+    }
+    """
+    storage = datastore()
+
+    existing_view: View = storage.view.get_if_exists(view_id)
+    if not existing_view:
+        return not_found(err="This view does not exist")
+
+    if existing_view.owner != user.uname and "admin" not in user.type:
+        return forbidden(err="You cannot delete a view unless you are an administrator, or the owner.")
+
+    if existing_view.type == "readonly":
+        return forbidden(err="You cannot delete built-in views.")
+
+    success = storage.view.delete(view_id)
+
+    storage.view.commit()
+
+    return no_content({"success": success})
+
+
+@generate_swagger_docs()
+@view_api.route("/<view_id>", methods=["PUT"])
+@api_login(required_priv=["R", "W"])
+def update_view(view_id: str, user: User, **kwargs):
+    """Update a view
+
+    Variables:
+    view_id => The view_id of the view to modify
+
+    Optional Arguments:
+    None
+
+    Data Block:
+    {
+        "title": "New View Name"    # The name of this view
+        "query": "howler.id:*"      # The query to run
+    }
+
+    Result Example:
+    {
+        ...view     # The updated view data
+    }
+    """
+    storage = datastore()
+
+    new_data = request.json
+    if not isinstance(new_data, dict):
+        return bad_request(err="Invalid data format")
+
+    if set(new_data.keys()) & {"view_id", "owner"}:
+        return bad_request(err="You cannot change the owner or id of a view.")
+
+    existing_view: View = storage.view.get_if_exists(view_id)
+    if not existing_view:
+        return not_found(err="This view does not exist")
+
+    if existing_view.type == "readonly":
+        return forbidden(err="You cannot edit a built-in view.")
+
+    if existing_view.type == "personal" and existing_view.owner != user.uname:
+        return forbidden(err="You cannot update a personal view that is not owned by you.")
+
+    if existing_view.type == "global" and existing_view.owner != user.uname and "admin" not in user.type:
+        return forbidden(err="Only the owner of a view and administrators can edit a global view.")
+
+    new_view = View(merge({}, existing_view.as_primitives(), new_data))
+
+    storage.view.save(new_view.view_id, new_view)
+
+    storage.view.commit()
+
+    try:
+        if "query" in new_data:
+            # Make sure the query is valid
+            storage.hit.search(new_data["query"])
+
+        return ok(storage.view.get_if_exists(existing_view.view_id, as_obj=False))
+    except SearchException:
+        return bad_request(err="You must use a valid query when updating a view.")
+    except HowlerException as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@view_api.route("/<view_id>/favourite", methods=["POST"])
+@api_login(required_priv=["R", "W"])
+def set_as_favourite(view_id: str, **kwargs):
+    """Add a view to a list of the user's favourites
+
+    Variables:
+    view_id => The id of the view to add as a favourite
+
+    Optional Arguments:
+    None
+
+    Data Block:
+    {}  # Empty
+
+    Result Example:
+    {
+        "success": True     # If the operation succeeded
+    }
+    """
+    storage = datastore()
+
+    existing_view: View = storage.view.get_if_exists(view_id)
+    if not existing_view:
+        return not_found(err="This view does not exist")
+
+    if existing_view.type != "global" and (
+        existing_view.owner != kwargs["user"]["uname"] and existing_view.owner != "none"
+    ):
+        return forbidden(err="You can only favourite global views, or views owned by you.")
+
+    try:
+        current_user = storage.user.get_if_exists(kwargs["user"]["uname"])
+
+        current_user["favourite_views"] = list(set(current_user.get("favourite_views", []) + [view_id]))
+
+        storage.user.save(current_user["uname"], current_user)
+
+        return ok()
+    except ValueError as e:
+        return bad_request(err=str(e))
+
+
+@generate_swagger_docs()
+@view_api.route("/<view_id>/favourite", methods=["DELETE"])
+@api_login(required_priv=["R", "W"])
+def remove_as_favourite(view_id: str, **kwargs):
+    """Remove a view from a list of the user's favourites
+
+    Variables:
+    id => The id of the view to remove as a favourite
+
+    Optional Arguments:
+    None
+
+    Result Example:
+    {
+        "success": True     # If the operation succeeded
+    }
+    """
+    storage = datastore()
+
+    try:
+        current_user = storage.user.get_if_exists(kwargs["user"]["uname"])
+
+        current_favourites: list[str] = current_user.get("favourite_views", [])
+
+        if view_id not in current_favourites:
+            return not_found(err="View is not favourited.")
+
+        current_user["favourite_views"] = [favourite for favourite in current_favourites if favourite != view_id]
+
+        storage.user.save(current_user["uname"], current_user)
+
+        return no_content()
+    except ValueError as e:
+        return bad_request(err=str(e))

howler/app.py

@@ -0,0 +1,235 @@
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from howler.plugins import get_plugins
+
+load_dotenv()
+
+# We append the plugin directory for howler to the python part
+PLUGIN_PATH = Path(os.environ.get("HWL_PLUGIN_DIRECTORY", "/etc/howler/plugins"))
+sys.path.insert(0, str(PLUGIN_PATH))
+
+from howler.odm.models.config import config
+
+if config.ui.debug and PLUGIN_PATH.exists():
+    for _plugin in PLUGIN_PATH.iterdir():
+        sys.path.append(
+            str(Path(os.path.realpath(_plugin)) / f"../.venv/lib/python3.{sys.version_info.minor}/site-packages")
+        )
+
+import logging
+from typing import Any, cast
+
+from authlib.integrations.flask_client import OAuth
+from elasticapm.contrib.flask import ElasticAPM
+from flasgger import Swagger
+from flask import Flask
+from flask.blueprints import Blueprint
+from flask.logging import default_handler
+from prometheus_client import make_wsgi_app
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
+
+from howler.api.base import api
+from howler.api.socket import socket_api
+from howler.api.v1 import apiv1
+from howler.api.v1.action import action_api
+from howler.api.v1.analytic import analytic_api
+from howler.api.v1.auth import auth_api
+from howler.api.v1.configs import config_api
+from howler.api.v1.dossier import dossier_api
+from howler.api.v1.help import help_api
+from howler.api.v1.hit import hit_api
+from howler.api.v1.overview import overview_api
+from howler.api.v1.search import search_api
+from howler.api.v1.template import template_api
+from howler.api.v1.tool import tool_api
+from howler.api.v1.user import user_api
+from howler.api.v1.view import view_api
+from howler.common.logging import get_logger
+from howler.config import (
+    DEBUG,
+    HWL_UNSECURED_UI,
+    HWL_USE_JOB_SYSTEM,
+    HWL_USE_REST_API,
+    HWL_USE_WEBSOCKET_API,
+    SECRET_KEY,
+    cache,
+    config,
+)
+from howler.cronjobs import setup_jobs
+from howler.error import errors
+from howler.healthz import healthz
+
+logger = get_logger(__file__)
+
+app = Flask(
+    "howler-api",
+    static_url_path="/api/static",
+    static_folder=config.ui.static_folder,
+)
+# Disable strict check on trailing slashes for endpoints
+app.url_map.strict_slashes = False
+app.config["JSON_SORT_KEYS"] = False
+
+app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {"/metrics": make_wsgi_app()})  # type: ignore[method-assign]
+
+swagger_template = {
+    "info": {
+        "title": "Howler API",
+        "description": (
+            "Howler is an application that allows analysts to triage hits and alerts. It provides a way for "
+            "analysts to efficiently review and analyze alerts generated by different analytics and detections."
+        ),
+    }
+}
+swagger = Swagger(
+    app,
+    template=swagger_template,
+    config={
+        "headers": [],
+        "static_url_path": "/api/swagger_static",
+        "specs": [
+            {
+                "endpoint": "apispec_v1",
+                "route": "/api/apispec_v1.json",
+                "rule_filter": lambda rule: True,  # all in
+                "model_filter": lambda tag: True,  # all in
+            }
+        ],
+        "specs_route": "/api/docs",
+    },
+)
+
+cache.init_app(app)
+
+app.logger.setLevel(60)  # This completely turns off the flask logger
+if HWL_UNSECURED_UI:
+    app.config.update(SESSION_COOKIE_SECURE=False, SECRET_KEY=SECRET_KEY, PREFERRED_URL_SCHEME="http")
+else:
+    app.config.update(SESSION_COOKIE_SECURE=True, SECRET_KEY=SECRET_KEY, PREFERRED_URL_SCHEME="https")
+
+app.register_blueprint(errors)
+app.register_blueprint(healthz)
+
+if HWL_USE_REST_API or DEBUG:
+    logger.debug("Enabled REST API")
+    app.register_blueprint(action_api)
+    app.register_blueprint(analytic_api)
+    app.register_blueprint(api)
+    app.register_blueprint(apiv1)
+    app.register_blueprint(auth_api)
+    app.register_blueprint(config_api)
+    app.register_blueprint(help_api)
+    app.register_blueprint(hit_api)
+    app.register_blueprint(search_api)
+    app.register_blueprint(template_api)
+    app.register_blueprint(overview_api)
+    app.register_blueprint(tool_api)
+    app.register_blueprint(user_api)
+    app.register_blueprint(view_api)
+    app.register_blueprint(dossier_api)
+
+    if config.core.notebook.enabled:
+        from howler.api.v1.notebook import notebook_api
+
+        logger.debug("Enabled Notebook Integration")
+        app.register_blueprint(notebook_api)
+
+    if config.core.borealis.enabled:
+        from howler.api.v1.borealis import borealis_api
+
+        logger.debug("Enabled Borealis Integration")
+        app.register_blueprint(borealis_api)
+
+    logger.info("Checking plugins for additional routes")
+    for plugin in get_plugins():
+        if not plugin.modules.routes:
+            continue
+
+        for route in cast(list[Blueprint], plugin.modules.routes):
+            logger.info("Enabling additional endpoint: %s", route.url_prefix)
+            app.register_blueprint(route)
+
+
+else:
+    logger.info("Disabled REST API")
+
+if HWL_USE_WEBSOCKET_API or DEBUG:
+    logger.debug("Enabled Websocket API")
+    app.register_blueprint(socket_api)
+else:
+    logger.info("Disabled Websocket API")
+
+if HWL_USE_JOB_SYSTEM or DEBUG:
+    setup_jobs()
+
+
+# Setup OAuth providers
+if config.auth.oauth.enabled:
+    providers = []
+    for name, provider in config.auth.oauth.providers.items():
+        p: dict[str, Any] = provider.model_dump()
+
+        # Set provider name
+        p["name"] = name
+
+        # Remove howler specific fields from oAuth config
+        p.pop("auto_create", None)
+        p.pop("auto_sync", None)
+        p.pop("user_get", None)
+        p.pop("auto_properties", None)
+        p.pop("uid_regex", None)
+        p.pop("uid_format", None)
+        p.pop("user_groups", None)
+        p.pop("user_groups_data_field", None)
+        p.pop("user_groups_name_field", None)
+        p.pop("app_provider", None)
+
+        # Add the provider to the list of providers
+        providers.append(p)
+
+    if providers:
+        oauth = OAuth()
+        for p in providers:
+            oauth.register(**p)
+        oauth.init_app(app)
+
+# Setup logging
+app.logger.setLevel(logger.getEffectiveLevel())
+app.logger.removeHandler(default_handler)
+if logger.parent:
+    for ph in logger.parent.handlers:
+        app.logger.addHandler(ph)
+
+# Setup APMs
+if config.core.metrics.apm_server.server_url is not None:
+    logger.info(f"Exporting application metrics to: {config.core.metrics.apm_server.server_url}")
+    ElasticAPM(
+        app,
+        server_url=config.core.metrics.apm_server.server_url,
+        service_name="howler_api",
+    )
+
+wlog = logging.getLogger("werkzeug")
+wlog.setLevel(logging.WARNING)
+if logger.parent:  # pragma: no cover
+    for h in logger.parent.handlers:
+        wlog.addHandler(h)
+
+
+def main():
+    """Main application function"""
+    app.jinja_env.cache = {}
+    app.run(
+        host="0.0.0.0",  # noqa: S104
+        debug=DEBUG,
+        port=int(os.getenv("FLASK_RUN_PORT", os.getenv("PORT", 5000))),
+        extra_files=os.environ.get("FLASK_RUN_EXTRA_FILES", "").split(":"),
+    )
+
+
+if __name__ == "__main__":
+    main()

howler/common/README.md

@@ -0,0 +1,144 @@
+# Utility Functions
+
+The `howler/common` folder provides the utility functions for the library. Each file inside this folder will be explained in this README.
+
+## chunk.py
+
+Has utilities to transform list of items into list of tuples grouping sets of X items together.
+
+- `chunk(list)`: The chunk function return a generator of tuples.
+
+- `chunked_list(list)`:
+
+    The chunked_list goes through all the items and returns a list of all the tuples.
+
+        chunked_list([1,2,3,4,5,6,7,8], 2): [(1,2), (3,4), (5,6), (7,8)]
+
+## classification.py
+
+This file, in conjunction to it's default configuration file `classification.yml`, provide support for handling classifications in the system (Access control). It is fully configurable and the configuration definition is provided in-line in the `classification.yml` file.
+
+### Classification object
+
+The classification object provides the different methods to parse, normalize and compare classification strings. Here are some notable functions you will likely be using:
+
+- `list_all_combinations()`: This function returns all possible classification strings that the current `classification.yml` file supports.
+- `get_access_control_parts()`: This functions splits the classification string in parts to be used in a lucene query.
+- `intersect_user_classification(c12n_1, c12n_2)`: This function takes two classification strings and generate the highest classification that both strings share in common.
+- `is_accessible(user_c12n, target_c12n)`: This function verifies if a user's maximum classification give them access to see a certain target classification.
+- `max_classification(c12n_1, c12n_2)`: This function returns the highest possible classification by mixing both classifications.
+- `min_classification(c12n_1, c12n_2)`: This function returns the minimum possible classification by mixing both classifications.
+
+## dict_utils.py
+
+This file provides utility functions to merge dictionaries together, find the differences between dictionaries or change the ways its keys are displayed.
+
+- `strip_nulls(dict)`: This function remove all keys that are null in the dictionary rescursively.
+- `recursive_update(dict, update_dict)`: This function recursively applied the update_dict values to the original dict that was provided
+- `get_recursive_delta(d1, d2)`: This function generate a delta dictionary that tells you which keys changed to which values if you go from d1 to d2.
+- `flatten/unflatten(dict)`:
+
+    The flatten function take a multiple level deep dictionary and transforms it into a single level dictionary by preserving the key space using a dotted notation:
+
+        {a: {b: 1} }: {a.b: 1}
+
+    Where as the unflatten does the invert by taking the dotted notation and transforming it back to it's original multiple level dictionary.
+
+## hexdump.py
+
+This file provide functions to take a binary data blob and transform it into and hexadecimal dump of its bytes. The `dump(buf)` function only outputs the bytes where as the `hexdump(buf)` function outputs also the offsets and some trimmed down ascii representation.
+
+`dump("HTTP/1.1 404 Not\r\nCont")`
+
+    48 54 54 50 2F 31 2E 31 20 34 30 34 20 4E 6F 74 20 46 6F 75 6E 64 0D 0A 43 6F 6E 74
+
+`hexdump("HTTP/1.1 404 Not\r\nCont")`
+
+    00000000:  48 54 54 50 2F 31 2E 31 20 34 30 34 20 4E 6F 74  HTTP/1.1 404 Not
+    00000010:  20 46 6F 75 6E 64 0D 0A 43 6F 6E 74              Found..Cont
+
+## iprange.py
+
+This file provides you with a RangeTable class that let's you determine if and IP is part of a certain CIDR definition. It also provides to quick function that let you determine if an IP is in a private CIDR (`is_private(ip)`) or if an IP is in a reserved CIDR (`is_reserved(ip)`).
+
+*Note*: only IPV4 IPs are supported.
+
+## isotime.py
+
+This file provides you which methods to transform date into strings or epoch values. It support local, ISO and epoch time. It also makes sure that the local and ISO time get up to a microsecond precision.
+
+Here are the support date operation:
+
+- Get current time functions:
+  - `now()` -> Current epoch time
+  - `now_as_iso()` -> Current iso time
+  - `now_as_local()` -> Current local time
+- Tranformation functions:
+  - `epoch_to_iso(date)`
+  - `epoch_to_local(date)`
+  - `local_to_epoch(date)`
+  - `local_to_iso(date)`
+  - `iso_to_epoch(date)`
+  - `iso_to_local(date)`
+
+## loader.py
+
+This file provide helper function for components that require external configuration files: Classification engine, datastore and remote datatypes.
+
+- `get_classification()`: returns a pre-configured classification object.
+- `get_config()`: returns the current classification of the system.
+- `get_datastore()`: returns an Howler datastore using the config form the get_config() output.
+
+## log.py/logformat.py
+
+This file provides an `init_logger()` function that will setup logging in your app using the configuration file and formats it using the format found in logformat.py
+
+## memory_zip.py
+
+Provides an interface file to create zip files in memory.
+
+## net.py/net_static.py
+
+Provide multiple function to validate ip/port/domains and the get networking information about the current host.
+
+- Validation:
+  - `is_valid_port(port)`
+  - `is_valid_domain(domain)`
+  - `is_valid_ip(ip)`
+  - `is_valid_email(email)`
+  - `is_ip_in_network(ip, cidr)`
+- Network information:
+  - `get_hostname()`
+  - `get_mac_address()`
+  - `get_route_to(ip)`
+  - `get_host_ip()`
+  - `get_host_default_gateway()`
+
+## random_user.py
+
+Generate random usernames base of a list of nouns and adjectives.
+
+## security.py
+
+Provide secure function to generate/validate passwords and API keys of users.
+
+- Password generation/validation:
+  - `get_password_hash(password)`: returns the hash of a plaintext password
+  - `verify_password(password, hash)`: verifies if a password matches a hash
+  - `get_random_password()`: generates a random password
+  - `check_password_requirements(password)`: Check if a password meets the minimum requirements
+
+## str_utils.py
+
+Provide functions to safely manipulate and transform strings.
+
+- `safe_str(buf)`: Make sure to safely encode bytes into uft-8 string or change the current string encoding to utf-8
+- `translate_str(buf)`: Try to guess the current encoding of a string or a byte buffer
+- `truncate(buf)`: Make sure a string does not exceed a certain length by adding ellipses.
+
+## uid.py
+
+Generate random ID in a format shorter then UUID and more double click friendly
+
+- `get_random_id()`: Generate a, base62 based + 22 character, collision free random ID
+- `get_id_from_data(data)`: Generate an ID base of the provided data