howler-api 2.10.0.dev131__tar.gz → 2.10.0.dev155__tar.gz

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: howler-api
-Version: 2.10.0.dev131
+Version: 2.10.0.dev155
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -50,7 +50,7 @@ Requires-Dist: python-datemath (==3.0.3)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
 Requires-Dist: pyyaml (==6.0.2)
 Requires-Dist: redis (==4.5.4)
-Requires-Dist: requests (==2.32.2)
+Requires-Dist: requests (==2.32.4)
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
 Requires-Dist: validators (>=0.34.0,<0.35.0)
 Requires-Dist: wsproto (==1.2.0)

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/actions/__init__.py

@@ -5,8 +5,8 @@ from pathlib import Path
 from typing import Any, Optional
 
 from howler.common.logging import get_logger
-from howler.config import config
 from howler.odm.models.user import User
+from howler.plugins import get_plugins
 
 logger = get_logger(__file__)
 
@@ -89,20 +89,25 @@ def execute(
     Returns:
         list[dict[str, Any]]: A report on the execution
     """
-    automation = None
-    for module_name in ["howler", *config.core.plugins]:
-        try:
-            automation = importlib.import_module(f"{module_name}.actions.{operation_id}")
-            break
-        except ImportError as err:
-            if f"No module named '{module_name}'" in str(err):
-                raise
-            else:
-                logger.info("Plugin %s does not expose operation %s.", module_name, operation_id)
-        except Exception as err:
-            logger.critical("Error when importing %s - %s", operation_id, err)
-
-    if not automation:
+    operation = None
+    try:
+        operation = importlib.import_module(f"howler.actions.{operation_id}")
+    except ImportError:
+        pass
+
+    if not operation:
+        for plugin in get_plugins():
+            if not plugin.modules.operations:
+                continue
+
+            operation = next(
+                (operation for operation in plugin.modules.operations if operation.OPERATION_ID == operation_id), None
+            )
+
+            if operation:
+                break
+
+    if not operation:
         return [
             {
                 "query": query,
@@ -112,7 +117,7 @@ def execute(
             }
         ]
 
-    missing_roles = set(automation.specification()["roles"]) - set(user["type"])
+    missing_roles = set(operation.specification()["roles"]) - set(user["type"])
     if missing_roles:
         return [
             {
@@ -126,7 +131,7 @@ def execute(
             }
         ]
 
-    report = automation.execute(query=query, request_id=request_id, user=user, **kwargs)
+    report = operation.execute(query=query, request_id=request_id, user=user, **kwargs)
 
     return __sanitize_report(report)
 
@@ -139,28 +144,24 @@ def specifications() -> list[dict[str, Any]]:
     """
     specifications = []
 
-    module_paths = {"howler": Path(__file__).parent}
-
-    for plugin in config.core.plugins:
-        plugin_module_path = PLUGIN_PATH / plugin / "actions"
-        if plugin_module_path.exists():
-            module_paths[plugin] = plugin_module_path
+    for module in (
+        _file
+        for _file in Path(__file__).parent.iterdir()
+        if _file.suffix == ".py" and _file.name not in ["__init__.py", "example_plugin.py"]
+    ):
+        try:
+            operation = importlib.import_module(f"howler.actions.{module.stem}")
 
-    for module_name, module_path in module_paths.items():
-        for module in (
-            _file
-            for _file in module_path.iterdir()
-            if _file.suffix == ".py" and _file.name not in ["__init__.py", "example_plugin.py"]
-        ):
-            try:
-                automation = importlib.import_module(f"{module_name}.actions.{module.stem}")
+            specifications.append(__sanitize_specification(operation.specification()))
 
-                if module_name != "howler":
-                    logger.info("Enabling action %s from plugin %s", automation.specification()["id"], module_name)
+        except Exception:  # pragma: no cover
+            logger.exception("Error when initializing %s", module)
 
-                specifications.append(__sanitize_specification(automation.specification()))
+    for plugin in get_plugins():
+        if not plugin.modules.operations:
+            continue
 
-            except Exception:  # pragma: no cover
-                logger.exception("Error when initializing %s", module)
+        for operation in plugin.modules.operations:
+            specifications.append(__sanitize_specification(operation.specification()))
 
     return specifications

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/api/v1/borealis.py

@@ -1,4 +1,4 @@
-import importlib
+import sys
 import time
 from typing import Callable, Optional
 
@@ -11,6 +11,7 @@ from howler.common.exceptions import AuthenticationException
 from howler.common.logging import get_logger
 from howler.common.swagger import generate_swagger_docs
 from howler.config import cache, config
+from howler.plugins import get_plugins
 from howler.security import api_login
 
 SUB_API = "borealis"
@@ -20,19 +21,19 @@ borealis_api._doc = "Proxy enrichment requests to borealis"
 logger = get_logger(__file__)
 
 
-@cache.memoize(15 * 60)
+def skip_cache(*args):
+    "Function to skip cache in testing mode"
+    return "pytest" in sys.modules
+
+
+@cache.memoize(15 * 60, unless=skip_cache)
 def get_token(access_token: str) -> str:
     """Get a borealis token based on the current howler token"""
     get_borealis_token: Optional[Callable[[str], str]] = None
 
-    for plugin in config.core.plugins:
-        try:
-            module = importlib.import_module(f"{plugin}.token.borealis")
-
-            get_borealis_token = module.get_borealis_token
+    for plugin in get_plugins():
+        if get_borealis_token := plugin.modules.token_functions.get("borealis", None):
             break
-        except ImportError:
-            logger.info("Plugin %s does not modify the borealis access token.")
 
     if get_borealis_token:
         borealis_access_token = get_borealis_token(access_token)

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/app.py

@@ -4,20 +4,22 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from howler.odm.models.config import config
+from howler.plugins import get_plugins
 
 load_dotenv()
 
 # We append the plugin directory for howler to the python part
 PLUGIN_PATH = Path(os.environ.get("HWL_PLUGIN_DIRECTORY", "/etc/howler/plugins"))
 sys.path.insert(0, str(PLUGIN_PATH))
+
+from howler.odm.models.config import config
+
 if config.ui.debug and PLUGIN_PATH.exists():
     for _plugin in PLUGIN_PATH.iterdir():
         sys.path.append(
             str(Path(os.path.realpath(_plugin)) / f"../.venv/lib/python3.{sys.version_info.minor}/site-packages")
         )
 
-import importlib
 import logging
 from typing import Any, cast
 
@@ -142,13 +144,14 @@ if HWL_USE_REST_API or DEBUG:
         logger.debug("Enabled Borealis Integration")
         app.register_blueprint(borealis_api)
 
-    for plugin in config.core.plugins:
-        try:
-            for route in cast(list[Blueprint], importlib.import_module(f"{plugin}.routes").ROUTES):
-                logger.info("Enabling additional endpoint: %s", route.url_prefix)
-                app.register_blueprint(route)
-        except ImportError:
-            logger.info("Plugin %s does not export additional endpoints.", plugin)
+    logger.info("Checking plugins for additional routes")
+    for plugin in get_plugins():
+        if not plugin.modules.routes:
+            continue
+
+        for route in cast(list[Blueprint], plugin.modules.routes):
+            logger.info("Enabling additional endpoint: %s", route.url_prefix)
+            app.register_blueprint(route)
 
 
 else:

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/cronjobs/__init__.py

@@ -25,4 +25,5 @@ def setup_jobs():
         except Exception as e:
             logger.critical("Error when initializing %s - %s", module, e)
 
-    scheduler.start()
+    if scheduler.state != 1:
+        scheduler.start()

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/datastore/collection.py

@@ -19,6 +19,7 @@ from datemath.helpers import DateMathException
 from howler import odm
 from howler.common.exceptions import HowlerRuntimeError, HowlerValueError, NonRecoverableError
 from howler.common.loader import APP_NAME
+from howler.common.logging.format import HWL_DATE_FORMAT, HWL_LOG_FORMAT
 from howler.datastore.bulk import ElasticBulkPlan
 from howler.datastore.constants import BACK_MAPPING, TYPE_MAPPING
 from howler.datastore.exceptions import (
@@ -58,6 +59,12 @@ if typing.TYPE_CHECKING:
 TRANSPORT_TIMEOUT = int(environ.get("HWL_DATASTORE_TRANSPORT_TIMEOUT", "10"))
 
 logger = logging.getLogger("howler.api.datastore")
+logger.setLevel(logging.INFO)
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+console.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+logger.addHandler(console)
+
 ModelType = TypeVar("ModelType", bound=Model)
 write_block_settings = {"settings": {"index.blocks.write": True}}
 write_unblock_settings = {"settings": {"index.blocks.write": None}}

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/datastore/howler_store.py

@@ -14,23 +14,39 @@ from howler.odm.models.overview import Overview
 from howler.odm.models.template import Template
 from howler.odm.models.user import User
 from howler.odm.models.view import View
+from howler.plugins import get_plugins
 
 if TYPE_CHECKING:
     from howler.datastore.store import ESStore
 
+INDEXES = [
+    ("hit", Hit),
+    ("template", Template),
+    ("overview", Overview),
+    ("analytic", Analytic),
+    ("action", Action),
+    ("user", User),
+    ("view", View),
+    ("dossier", Dossier),
+    ("user_avatar", None),
+]
+
 
 class HowlerDatastore(object):
     def __init__(self, datastore_object: "ESStore"):
         self.ds = datastore_object
-        self.ds.register("hit", Hit)
-        self.ds.register("template", Template)
-        self.ds.register("overview", Overview)
-        self.ds.register("analytic", Analytic)
-        self.ds.register("action", Action)
-        self.ds.register("user", User)
-        self.ds.register("view", View)
-        self.ds.register("dossier", Dossier)
-        self.ds.register("user_avatar")
+
+        for plugin in get_plugins():
+            for _index, _odm in INDEXES:
+                if _odm is None:
+                    continue
+
+                if modify_odm := plugin.modules.odm.modify_odm.get(_index):
+                    logger.info("Modifying %s odm with function from plugin %s", _index, plugin.name)
+                    modify_odm(_odm)
+
+        for _index, _odm in INDEXES:
+            self.ds.register(_index, _odm)
 
     def __enter__(self):
         return self

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/helper/search.py

@@ -11,17 +11,15 @@ ADMIN_INDEX_MAP: dict[str, Callable[[], ESCollection]] = {}
 
 ADMIN_INDEX_ORDER_MAP: dict[str, str] = {}
 
-ds = datastore()
-
 INDEX_MAP: dict[str, Callable[[], ESCollection]] = {
-    "action": lambda: ds.action,
-    "analytic": lambda: ds.analytic,
-    "dossier": lambda: ds.dossier,
-    "hit": lambda: ds.hit,
-    "overview": lambda: ds.overview,
-    "template": lambda: ds.template,
-    "user": lambda: ds.user,
-    "view": lambda: ds.view,
+    "action": lambda: datastore().action,
+    "analytic": lambda: datastore().analytic,
+    "dossier": lambda: datastore().dossier,
+    "hit": lambda: datastore().hit,
+    "overview": lambda: datastore().overview,
+    "template": lambda: datastore().template,
+    "user": lambda: datastore().user,
+    "view": lambda: datastore().view,
 }
 
 INDEX_ORDER_MAP: dict[str, str] = {

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/odm/helper.py

@@ -1,4 +1,3 @@
-import importlib
 import json
 import random
 import sys
@@ -28,6 +27,7 @@ from howler.odm.randomizer import (
     random_department,
     random_model_obj,
 )
+from howler.plugins import get_plugins
 from howler.security.utils import get_password_hash
 from howler.utils.uid import get_random_id
 
@@ -285,12 +285,10 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
         log.previous_version = get_random_id()
 
     new_keys: list[str] = []
-    for plugin in config.core.plugins:
-        try:
-            _new_keys, hit = importlib.import_module(f"{plugin}.odm.hit").generate_useful_hit(hit)
+    for plugin in get_plugins():  # pragma: no cover
+        if generate := plugin.modules.odm.generation.get("hit", None):
+            _new_keys, hit = generate(hit)
             new_keys += _new_keys
-        except (ImportError, AttributeError):
-            logger.exception("Plugin does not expose useful hit generation")
 
     if len(new_keys) > 0:
         logger.debug("%s new top-level fields configured")

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/odm/models/hit.py

@@ -1,10 +1,8 @@
 # mypy: ignore-errors
-import importlib
 from typing import Optional
 
 from howler import odm
 from howler.common.logging import get_logger
-from howler.config import config
 from howler.odm.models.assemblyline import AssemblyLine
 from howler.odm.models.aws import AWS
 from howler.odm.models.azure import Azure
@@ -351,16 +349,6 @@ class Hit(odm.Model):
     )
 
 
-for plugin in config.core.plugins:
-    try:
-        importlib.import_module(f"{plugin}.odm.hit").modify_odm(Hit)
-    except (ImportError, AttributeError) as err:
-        if f"No module named '{plugin}'" in str(err):
-            raise
-        else:
-            logger.info("Plugin %s does not modify the ODM.", plugin)
-
-
 if __name__ == "__main__":
     from pprint import pprint
 

{howler_api-2.10.0.dev131 → howler_api-2.10.0.dev155}/howler/odm/random_data.py

@@ -4,6 +4,8 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
+from howler.plugins import get_plugins
+
 load_dotenv()
 
 # We append the plugin directory for howler to the python part
@@ -50,6 +52,20 @@ logger = get_logger(__file__)
 hit_helper = OdmHelper(Hit)
 
 
+def run_modifications(odm: str, data: Any, log: bool = False):
+    "Running modifications"
+    new_keys: list[str] = []
+    for plugin in get_plugins():  # pragma: no cover
+        if generate := plugin.modules.odm.generation.get(odm, None):
+            _new_keys, data = generate(data)
+            new_keys += _new_keys
+
+    if len(new_keys) > 0 and log:
+        logger.debug("%s new top-level fields configured for %s", len(new_keys), odm)
+
+    return data
+
+
 def create_users(ds):
     """Create  number of user accounts"""
     admin_pass = os.getenv("DEV_ADMIN_PASS", "admin") or "admin"
@@ -68,6 +84,9 @@ def create_users(ds):
             "owner": "admin",
         }
     )
+
+    admin_view = run_modifications("view", admin_view)
+
     user_data = User(
         {
             "apikeys": {
@@ -106,6 +125,9 @@ def create_users(ds):
             "favourite_views": [admin_view.view_id],
         }
     )
+
+    user_data = run_modifications("view", user_data, True)
+
     ds.user.save("admin", user_data)
     ds.user_avatar.save(
         "admin",
@@ -149,6 +171,10 @@ def create_users(ds):
             "favourite_views": [user_view.view_id],
         }
     )
+
+    user_view = run_modifications("view", user_view)
+    user_data = run_modifications("user", user_data)
+
     ds.user.save("user", user_data)
     ds.user_avatar.save(
         "user",
@@ -192,6 +218,10 @@ def create_users(ds):
             "favourite_views": [huey_view.view_id],
         }
     )
+
+    huey_view = run_modifications("view", huey_view)
+    huey_data = run_modifications("user", huey_data)
+
     ds.user.save("huey", huey_data)
     ds.user_avatar.save(
         "huey",
@@ -223,7 +253,9 @@ def create_users(ds):
         }
     )
 
-    shawn_data.favourite_views.append(shawnh_view.view_id)
+    shawnh_view = run_modifications("view", shawnh_view)
+    shawn_data = run_modifications("user", shawn_data)
+
     ds.user.save("shawn-h", shawn_data)
     ds.view.save(shawnh_view.view_id, shawnh_view)
 
@@ -251,7 +283,9 @@ def create_users(ds):
         }
     )
 
-    goose_data.favourite_views.append(goose_view.view_id)
+    goose_view = run_modifications("view", goose_view)
+    goose_data = run_modifications("user", goose_data)
+
     ds.user.save("goose", goose_data)
     ds.view.save(goose_view.view_id, goose_view)
 
@@ -271,7 +305,7 @@ def wipe_users(ds):
 
 def create_templates(ds: HowlerDatastore):
     """Create some random templates"""
-    for _ in range(2):
+    for i in range(2):
         keys = sample(list(Hit.flat_fields().keys()), 5)
 
         for detection in ["Detection 1", "Detection 2"]:
@@ -284,6 +318,8 @@ def create_templates(ds: HowlerDatastore):
                 }
             )
 
+            template = run_modifications("template", template, i == 0)
+
             ds.template.save(
                 template.template_id,
                 template,
@@ -298,6 +334,8 @@ def create_templates(ds: HowlerDatastore):
             }
         )
 
+        template = run_modifications("template", template)
+
         ds.template.save(
             template.template_id,
             template,
@@ -312,6 +350,8 @@ def create_templates(ds: HowlerDatastore):
             }
         )
 
+        template = run_modifications("template", template)
+
         ds.template.save(
             template.template_id,
             template,
@@ -341,7 +381,7 @@ def wipe_templates(ds):
 
 def create_overviews(ds: HowlerDatastore):
     """Create some random overviews"""
-    for _ in range(2):
+    for i in range(2):
         keys = sample(list(Hit.flat_fields().keys()), 5)
 
         for detection in ["Detection 1", "Detection 2"]:
@@ -355,6 +395,8 @@ def create_overviews(ds: HowlerDatastore):
                 }
             )
 
+            overview = run_modifications("overview", overview, i == 0)
+
             ds.overview.save(
                 overview.overview_id,
                 overview,
@@ -399,6 +441,8 @@ def create_overviews(ds: HowlerDatastore):
             }
         )
 
+        overview = run_modifications("overview", overview)
+
         ds.overview.save(
             overview.overview_id,
             overview,
@@ -423,6 +467,8 @@ def create_views(ds: HowlerDatastore):
         }
     )
 
+    view = run_modifications("view", view)
+
     ds.view.save(
         view.view_id,
         view,
@@ -437,6 +483,8 @@ def create_views(ds: HowlerDatastore):
         }
     )
 
+    view = run_modifications("view", view)
+
     ds.view.save(
         view.view_id,
         view,
@@ -455,6 +503,8 @@ def create_views(ds: HowlerDatastore):
             }
         )
 
+        view = run_modifications("view", view)
+
         ds.view.save(
             view.view_id,
             view,
@@ -598,6 +648,8 @@ def create_analytics(ds: HowlerDatastore, num_analytics: int = 10):
                 )
             )
 
+        analytic = run_modifications("analytic", analytic)
+
         ds.analytic.save(analytic.analytic_id, analytic)
 
     fields = Hit.flat_fields()
@@ -618,6 +670,8 @@ def create_analytics(ds: HowlerDatastore, num_analytics: int = 10):
             set(random.sample(assessments, counts=([3] * len(assessments)), k=random.randint(1, len(assessments) * 3)))
         )
 
+        a = run_modifications("analytic", a)
+
         ds.analytic.save(a.analytic_id, a)
 
     for rule_type in ["lucene", "eql", "sigma"]:
@@ -670,6 +724,8 @@ def create_analytics(ds: HowlerDatastore, num_analytics: int = 10):
                     "For better test data using sigma rules, execute howler/external/generate_sigma_rules.py."
                 )
 
+        a = run_modifications("analytic", a)
+
         ds.analytic.save(a.analytic_id, a)
 
     ds.analytic.commit()
@@ -735,6 +791,8 @@ def create_actions(ds: HowlerDatastore, num_actions: int = 30):
             }
         )
 
+        action = run_modifications("action", action)
+
         ds.action.save(action.action_id, action)
 
     ds.action.commit()

howler_api-2.10.0.dev155/howler/plugins/__init__.py

@@ -0,0 +1,25 @@
+import importlib
+from typing import Optional
+
+from howler.common.logging import get_logger
+from howler.config import config as _config  # Python gets BIG mad if we don't alias this
+from howler.plugins.config import BasePluginConfig
+
+logger = get_logger(__file__)
+
+PLUGINS: dict[str, Optional[BasePluginConfig]] = {}
+
+
+def get_plugins() -> list[BasePluginConfig]:
+    "Get a set of plugin configurations based on the howler settings."
+    for plugin in _config.core.plugins:
+        if plugin in PLUGINS:
+            continue
+
+        try:
+            PLUGINS[plugin] = importlib.import_module(f"{plugin}.config").config
+        except (ImportError, ModuleNotFoundError):
+            logger.exception("Exception when loading plugin %s", plugin)
+            PLUGINS[plugin] = None
+
+    return [plugin for plugin in PLUGINS.values() if plugin]