howler-api 2.13.0.dev341__tar.gz → 2.13.0.dev343__tar.gz

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 2.13.0.dev341
+Version: 2.13.0.dev343
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/api/__init__.py

@@ -32,7 +32,7 @@ def _make_api_response(
 ) -> Response:
     quota_user = flsk_session.pop("quota_user", None)
     quota_set = flsk_session.pop("quota_set", False)
-    if quota_user and quota_set and not request.path.startswith("/api/v1/borealis"):
+    if quota_user and quota_set and not request.path.startswith("/api/v1/clue"):
         QUOTA_TRACKER.end(quota_user)
 
     if type(err) is Exception:  # pragma: no cover

howler_api-2.13.0.dev341/howler/api/v1/borealis.py → howler_api-2.13.0.dev343/howler/api/v1/clue.py

@@ -14,9 +14,9 @@ from howler.config import cache, config
 from howler.plugins import get_plugins
 from howler.security import api_login
 
-SUB_API = "borealis"
-borealis_api = make_subapi_blueprint(SUB_API, api_version=1)
-borealis_api._doc = "Proxy enrichment requests to borealis"
+SUB_API = "clue"
+clue_api = make_subapi_blueprint(SUB_API, api_version=1)
+clue_api._doc = "Proxy enrichment requests to clue"
 
 logger = get_logger(__file__)
 
@@ -28,27 +28,27 @@ def skip_cache(*args):
 
 @cache.memoize(15 * 60, unless=skip_cache)
 def get_token(access_token: str) -> str:
-    """Get a borealis token based on the current howler token"""
-    get_borealis_token: Optional[Callable[[str], str]] = None
+    """Get a clue token based on the current howler token"""
+    get_clue_token: Optional[Callable[[str], str]] = None
 
     for plugin in get_plugins():
-        if get_borealis_token := plugin.modules.token_functions.get("borealis", None):
+        if get_clue_token := plugin.modules.token_functions.get("clue", None):
             break
 
-    if get_borealis_token:
-        borealis_access_token = get_borealis_token(access_token)
+    if get_clue_token:
+        clue_access_token = get_clue_token(access_token)
     else:
-        logger.info("No custom borealis token logic provided, continuing with howler credentials")
-        borealis_access_token = access_token
+        logger.info("No custom clue token logic provided, continuing with howler credentials")
+        clue_access_token = access_token
 
-    return borealis_access_token
+    return clue_access_token
 
 
 @generate_swagger_docs()
-@borealis_api.route("/<path:path>", methods=["GET", "POST"])
+@clue_api.route("/<path:path>", methods=["GET", "POST"])
 @api_login(required_priv=["R"], required_method=["oauth"])
-def proxy_to_borealis(path, **kwargs):
-    """Proxy enrichment requests to Borealis
+def proxy_to_clue(path, **kwargs):
+    """Proxy enrichment requests to Clue
 
     Variables:
     None
@@ -60,11 +60,9 @@ def proxy_to_borealis(path, **kwargs):
     Any
 
     Result Example:
-    Borealis Responses
+    Clue Responses
     """
-    logger.info(
-        "Proxying borealis request to path %s/%s?%s", config.core.borealis.url, path, request.query_string.decode()
-    )
+    logger.info("Proxying clue request to path %s/%s?%s", config.core.clue.url, path, request.query_string.decode())
 
     auth_data: Optional[str] = request.headers.get("Authorization", None, type=str)
 
@@ -73,29 +71,29 @@ def proxy_to_borealis(path, **kwargs):
 
     auth_token = auth_data.split(" ")[1]
 
-    borealis_token = get_token(auth_token)
+    clue_token = get_token(auth_token)
 
     start = time.perf_counter()
-    with elasticapm.capture_span("borealis", span_type="http"):
+    with elasticapm.capture_span("clue", span_type="http"):
         if request.method.lower() == "get":
             response = requests.get(
-                f"{config.core.borealis.url}/{path}",
-                headers={"Authorization": f"Bearer {borealis_token}", "Accept": "application/json"},
+                f"{config.core.clue.url}/{path}",
+                headers={"Authorization": f"Bearer {clue_token}", "Accept": "application/json"},
                 params=request.args.to_dict(),
                 timeout=5 * 60,
             )
         else:
             response = requests.post(
-                f"{config.core.borealis.url}/{path}",
+                f"{config.core.clue.url}/{path}",
                 json=request.json,
-                headers={"Authorization": f"Bearer {borealis_token}", "Accept": "application/json"},
+                headers={"Authorization": f"Bearer {clue_token}", "Accept": "application/json"},
                 params=request.args.to_dict(),
                 timeout=5 * 60,
             )
 
-    logger.debug(f"Request to borealis completed in {round(time.perf_counter() - start)}ms")
+    logger.debug(f"Request to clue completed in {round(time.perf_counter() - start)}ms")
 
     if not response.ok:
-        return bad_gateway(response.json(), err="Something went wrong when connecting to borealis")
+        return bad_gateway(response.json(), err="Something went wrong when connecting to clue")
 
     return ok(response.json()["api_response"])

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/api/v1/search.py

@@ -1,6 +1,9 @@
+import re
+from copy import deepcopy
 from typing import Any, Union
 
 from elasticsearch import BadRequestError
+from elasticsearch._sync.client.indices import IndicesClient
 from flask import request
 from sigma.backends.elasticsearch import LuceneBackend
 from sigma.rule import SigmaRule
@@ -14,7 +17,7 @@ from howler.common.swagger import generate_swagger_docs
 from howler.datastore.exceptions import SearchException
 from howler.helper.search import get_collection, get_default_sort, has_access_control, list_all_fields
 from howler.security import api_login
-from howler.services import hit_service
+from howler.services import hit_service, lucene_service
 
 SUB_API = "search"
 search_api = make_subapi_blueprint(SUB_API, api_version=1)
@@ -146,6 +149,76 @@ def search(index, **kwargs):
         return bad_request(err=f"SearchException: {e}")
 
 
+@generate_swagger_docs()
+@search_api.route("/<index>/explain", methods=["GET", "POST"])
+@api_login(required_priv=["R"])
+def explain_query(index, **kwargs):
+    """Search through specified index for a given Lucene query. Uses Lucene search syntax for query.
+
+    Variables:
+    index  =>   Index to explain against (hit, user,...)
+
+    Arguments:
+    query   =>   Lucene Query to explain
+
+    Data Block:
+    # Note that the data block is for POST requests only!
+    {
+        "query": "id:*", # Lucene Query to explain
+    }
+
+
+    Result Example:
+    {
+        'valid': True,
+        'explanations': [
+            {
+                'valid': True,
+                'explanation': 'ConstantScore(FieldExistsQuery [field=id])'
+            }
+        ]
+    }
+    """
+    user = kwargs["user"]
+    collection = get_collection(index, user)
+
+    if collection is None:
+        return bad_request(err=f"Not a valid index to explain: {index}")
+
+    fields = ["query"]
+    multi_fields: list[str] = []
+
+    params, req_data = generate_params(request, fields, multi_fields)
+
+    params["as_obj"] = False
+
+    query = req_data.get("query", None)
+    if not query:
+        return bad_request(err="There was no query.")
+
+    # This regex checks for lucene phrases (i.e. the "Example Analytic" part of howler.analytic:"Example Analytic")
+    # And then escapes them.
+    # https://regex101.com/r/8u5F6a/1
+    escaped_lucene = re.sub(r'((:\()?(".+?")(\)?))', lucene_service.replace_lucene_phrase, query)
+
+    try:
+        indices_client = IndicesClient(datastore().hit.datastore.client)
+
+        result = deepcopy(
+            indices_client.validate_query(q=escaped_lucene, explain=True, index=collection().index_name).body
+        )
+
+        del result["_shards"]
+
+        for explanation in result["explanations"]:
+            del explanation["index"]
+
+        return ok(result)
+    except Exception as e:
+        logger.exception("Exception on query explanation")
+        return bad_request(err=f"Exception: {e}")
+
+
 @generate_swagger_docs()
 @search_api.route("/<index>/eql", methods=["GET", "POST"])
 @api_login(required_priv=["R"])

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/app.py

@@ -138,11 +138,11 @@ if HWL_USE_REST_API or DEBUG:
         logger.debug("Enabled Notebook Integration")
         app.register_blueprint(notebook_api)
 
-    if config.core.borealis.enabled:
-        from howler.api.v1.borealis import borealis_api
+    if config.core.clue.enabled:
+        from howler.api.v1.clue import clue_api
 
-        logger.debug("Enabled Borealis Integration")
-        app.register_blueprint(borealis_api)
+        logger.debug("Enabled Clue Integration")
+        app.register_blueprint(clue_api)
 
     logger.info("Checking plugins for additional routes")
     for plugin in get_plugins():

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/odm/helper.py

@@ -259,13 +259,13 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
         ),
     ]
 
-    if config.core.borealis.enabled:
+    if config.core.clue.enabled:
         hit.howler.dossier.append(
             Lead(
                 {
                     "icon": "material-symbols:image",
-                    "label": {"en": "Borealis", "fr": "Borealis"},
-                    "format": "borealis",
+                    "label": {"en": "Clue", "fr": "Clue"},
+                    "format": "clue",
                     "content": "test-plugin.image",
                     "metadata": {"type": "ip", "value": "127.0.01", "classification": "TLP:CLEAR"},
                 }
@@ -276,8 +276,8 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
             Lead(
                 {
                     "icon": "material-symbols:code-rounded",
-                    "label": {"en": "Borealis", "fr": "Borealis"},
-                    "format": "borealis",
+                    "label": {"en": "Clue", "fr": "Clue"},
+                    "format": "clue",
                     "content": "test-plugin.json",
                     "metadata": {"type": "ip", "value": "127.0.01", "classification": "TLP:CLEAR"},
                 }

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/odm/models/config.py

@@ -409,24 +409,24 @@ class UI(BaseModel):
     )
 
 
-class Borealis(BaseModel):
-    """Borealis enrichment service integration configuration.
+class Clue(BaseModel):
+    """Clue enrichment service integration configuration.
 
-    Defines settings for integrating with Borealis, an external enrichment
+    Defines settings for integrating with Clue, an external enrichment
     service that can provide additional context and status information for
     hits displayed in the Howler UI.
     """
 
-    enabled: bool = Field(default=False, description="Should borealis integration be enabled?")
+    enabled: bool = Field(default=False, description="Should clue integration be enabled?")
 
     url: str = Field(
         default="http://enrichment-rest.enrichment.svc.cluster.local:5000",
-        description="What url should Howler connect to to interact with Borealis?",
+        description="What url should Howler connect to to interact with Clue?",
     )
 
     status_checks: list[str] = Field(
         default=[],
-        description="A list of borealis fetchers that return status results given a Howler ID to show in the UI.",
+        description="A list of clue fetchers that return status results given a Howler ID to show in the UI.",
     )
 
 
@@ -451,7 +451,7 @@ class Core(BaseModel):
     """Core application configuration for Howler.
 
     Aggregates all core service configurations including Redis, metrics,
-    and external integrations like Borealis and nbgallery notebooks.
+    and external integrations like Clue and nbgallery notebooks.
     Also manages the loading of external plugins.
     """
 
@@ -463,8 +463,8 @@ class Core(BaseModel):
     redis: Redis = Redis()
     "Configuration for Redis instances"
 
-    borealis: Borealis = Borealis()
-    "Configuration for Borealis Integration"
+    clue: Clue = Clue()
+    "Configuration for Clue Integration"
 
     notebook: Notebook = Notebook()
     "Configuration for Notebook Integration"
@@ -529,7 +529,7 @@ class Config(BaseSettings):
     logging: Logging = Logging()
     system: System = System()
     ui: UI = UI()
-    mapping: dict[str, str] = Field(description="Mapping of alert keys to borealis type", default={})
+    mapping: dict[str, str] = Field(description="Mapping of alert keys to clue types", default={})
 
     model_config = SettingsConfigDict(
         yaml_file=config_locations,

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/odm/models/pivot.py

@@ -31,7 +31,7 @@ class Pivot(odm.Model):
         description="An optional icon to use in the tab display for this dossier.", optional=True
     )
     label: LocalizedLabel = odm.Compound(LocalizedLabel, description="Labels for the pivot in the UI.")
-    value: str = odm.Keyword(description="The link/borealis id to pivot on.")
+    value: str = odm.Keyword(description="The link/plugin information to pivot on.")
     format: str = odm.Keyword(description="The format of the pivot.")
     mappings: list[Mapping] = odm.List(
         odm.Compound(Mapping),

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/security/__init__.py

@@ -220,8 +220,8 @@ class api_login(object):  # noqa: D101, N801
                     user_id=user.get("uname", None),
                 )
 
-            if request.path.startswith("/api/v1/borealis"):
-                logger.debug("Bypassing quota limits for borealis enrichment")
+            if request.path.startswith("/api/v1/clue"):
+                logger.debug("Bypassing quota limits for clue enrichment")
             elif self.enforce_quota:
                 # Check current user quota
                 flsk_session["quota_user"] = user["uname"]

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/howler/services/config_service.py

@@ -117,11 +117,11 @@ def get_configuration(user: User, **kwargs):
             },
             "mapping": config.mapping,
             "features": {
-                "borealis": config.core.borealis.enabled,
+                "clue": config.core.clue.enabled,
                 "notebook": config.core.notebook.enabled,
                 **plugin_features,
             },
-            "borealis": {"status_checks": config.core.borealis.status_checks},
+            "clue": {"status_checks": config.core.clue.status_checks},
         },
         "c12nDef": classification_definition,
         "indexes": list_all_fields("admin" in user["type"] if user is not None else False),

{howler_api-2.13.0.dev341 → howler_api-2.13.0.dev343}/pyproject.toml

@@ -148,7 +148,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "2.13.0.dev341"
+version = "2.13.0.dev343"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",