howler-api 3.4.0.dev927__tar.gz → 3.4.0.dev933__tar.gz

{howler_api-3.4.0.dev927 → howler_api-3.4.0.dev933}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 3.4.0.dev927
+Version: 3.4.0.dev933
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-3.4.0.dev927 → howler_api-3.4.0.dev933}/howler/api/v1/hit.py

@@ -123,7 +123,7 @@ def create_hits(user: User, **kwargs):
 
             datastore().hit.commit()
 
-            action_service.bulk_execute_on_query(f"howler.id:({' OR '.join(odm.howler.id for odm in odms)})", user=user)
+            action_service.enqueue_action_execution([odm.howler.id for odm in odms], trigger="create", user=user)
 
         response_body["warnings"] = warnings
 
@@ -551,11 +551,7 @@ def add_label(id, label_set, user, **kwargs):
 
     datastore().hit.commit()
 
-    action_service.bulk_execute_on_query(
-        f"howler.id:{id}",
-        trigger="add_label",
-        user=user,
-    )
+    action_service.enqueue_action_execution([id], trigger="add_label", user=user)
 
     hit, version = hit_service.get_hit(id, as_odm=False, version=True)
 
@@ -609,11 +605,7 @@ def remove_labels(id, label_set, user, **kwargs):
 
     datastore().hit.commit()
 
-    action_service.bulk_execute_on_query(
-        f"howler.id:{id}",
-        trigger="remove_label",
-        user=user,
-    )
+    action_service.enqueue_action_execution([id], trigger="remove_label", user=user)
 
     hit, version = hit_service.get_hit(id, as_odm=False, version=True)
 

{howler_api-3.4.0.dev927 → howler_api-3.4.0.dev933}/howler/api/v1/search.py

@@ -78,6 +78,7 @@ def search(index, **kwargs):
     sort                =>   How to sort the results (not available in deep paging)
     fl                  =>   List of fields to return
     timeout             =>   Maximum execution time (ms)
+    use_archive         =>   Allow access to the datastore achive (Default: False)
     track_total_hits    =>   Track the total number of query matches, instead of stopping at 10000 (Default: False)
     metadata            =>   A list of additional features to be added to the result alongside the raw results
 
@@ -117,9 +118,18 @@ def search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters", "metadata"]
+    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
+    params.update(
+        {
+            k: str(req_data.get(k, "false")).lower() in ["true", ""]
+            for k in boolean_fields
+            if req_data.get(k, None) is not None
+        }
+    )
+
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -340,9 +350,18 @@ def sigma_search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters"]
+    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
+    params.update(
+        {
+            k: str(req_data.get(k, "false")).lower() in ["true", ""]
+            for k in boolean_fields
+            if req_data.get(k, None) is not None
+        }
+    )
+
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -501,6 +520,7 @@ def count(index, **kwargs):
     Optional Arguments:
     filters             =>   List of additional filter queries limit the data
     timeout             =>   Maximum execution time (ms)
+    use_archive         =>   Allow access to the datastore achive (Default: False)
 
     Data Block:
     # Note that the data block is for POST requests only!
@@ -523,6 +543,15 @@ def count(index, **kwargs):
 
     params, req_data = generate_params(request, [], [])
 
+    boolean_fields = ["use_archive"]
+    params.update(
+        {
+            k: str(req_data.get(k, "false")).lower() in ["true", ""]
+            for k in boolean_fields
+            if req_data.get(k, None) is not None
+        }
+    )
+
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 

{howler_api-3.4.0.dev927 → howler_api-3.4.0.dev933}/howler/api/v1/tool.py

@@ -178,6 +178,6 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
 
         datastore().hit.commit()
 
-        action_service.bulk_execute_on_query(f"howler.id:({' OR '.join(entry['id'] for entry in out)})", user=user)
+        action_service.enqueue_action_execution([entry["id"] for entry in out], trigger="create", user=user)
 
         return created(out, warnings=warnings)

howler_api-3.4.0.dev933/howler/cronjobs/action_queue_worker.py

@@ -0,0 +1,82 @@
+"""Action queue worker cronjob — starts per-trigger action queue consumer threads.
+
+Auto-discovered by ``howler.cronjobs.setup_jobs`` when ``HWL_USE_JOB_SYSTEM``
+is enabled.  Instead of scheduling a periodic APScheduler job, this module
+starts a long-running daemon thread per trigger type that drains its
+dedicated action queue.
+"""
+
+import threading
+
+from apscheduler.schedulers.base import BaseScheduler
+
+from howler.common.logging import get_logger
+from howler.config import config
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.services.action_service import _get_action_queue, process_action_batch
+
+logger = get_logger(__file__)
+
+_threads: dict[str, threading.Thread] = {}
+
+BATCH_SIZE: int = config.system.action_queue.batch_size
+BATCH_TIMEOUT: int = config.system.action_queue.batch_timeout
+
+
+def run_worker(trigger: str) -> None:  # pragma: no cover – long-running loop, tested via process_action_batch
+    """Block on the action queue for *trigger* and process batches.
+
+    Accumulates up to ``BATCH_SIZE`` items or flushes after ``BATCH_TIMEOUT``
+    seconds, whichever comes first.
+    """
+    if not config.system.action_queue.enabled:
+        logger.info("Action queue worker disabled by configuration, not starting for trigger=%s", trigger)
+        return
+
+    queue = _get_action_queue(trigger)
+    logger.info(
+        "Action queue worker started for trigger=%s (batch_size=%d, timeout=%ds)",
+        trigger,
+        BATCH_SIZE,
+        BATCH_TIMEOUT,
+    )
+
+    batch: list[dict] = []
+
+    while True:
+        try:
+            item = queue.pop(blocking=True, timeout=BATCH_TIMEOUT)
+
+            if item is not None:
+                batch.append(item)
+
+            if len(batch) >= BATCH_SIZE or (item is None and batch):
+                logger.debug("Processing action batch of %d item(s) for trigger=%s", len(batch), trigger)
+                try:
+                    process_action_batch(trigger, batch)
+                    logger.info("Action batch complete: %d item(s) processed for trigger=%s", len(batch), trigger)
+                except Exception:
+                    logger.exception("Error processing action batch for trigger=%s", trigger)
+                finally:
+                    batch = []
+        except Exception:
+            logger.exception("Unexpected error in action queue worker loop for trigger=%s", trigger)
+
+
+def setup_job(sched: BaseScheduler):
+    """Start an action queue worker thread per trigger type if the action queue is enabled."""
+    global _threads
+
+    if not config.system.action_queue.enabled:
+        logger.info("Action queue worker disabled by configuration")
+        return
+
+    for trigger in VALID_TRIGGERS:
+        if trigger in _threads and _threads[trigger].is_alive():
+            logger.debug("Action queue worker thread for trigger=%s already running", trigger)
+            continue
+
+        thread = threading.Thread(target=run_worker, args=(trigger,), name=f"action-queue-{trigger}", daemon=True)
+        thread.start()
+        _threads[trigger] = thread
+        logger.info("Action queue worker thread started for trigger=%s", trigger)

howler_api-3.4.0.dev933/howler/cronjobs/retention.py

@@ -0,0 +1,129 @@
+import os
+from datetime import datetime, timedelta
+from typing import Any
+
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.triggers.cron import CronTrigger
+from pytz import timezone
+
+from howler.common.logging import get_logger
+from howler.config import DEBUG, config
+from howler.datastore.howler_store import HowlerDatastore
+
+logger = get_logger(__file__)
+
+
+def execute():
+    """Delete any hits older than the configured time"""
+    from howler.common.loader import datastore
+
+    delta_kwargs = {str(config.system.retention.limit_unit): config.system.retention.limit_amount}
+
+    cutoff = (datetime.now() - timedelta(**delta_kwargs)).strftime("%Y-%m-%d")
+
+    logger.debug("Removing hits older than %s", cutoff)
+
+    ds = datastore()
+
+    ds.hit.delete_by_query(f"event.created:{{* TO {cutoff}}} OR howler.expiry:{{* TO now}}")
+
+    ds.hit.commit()
+
+    logger.debug("Deletion complete")
+
+    logger.debug("Cleaning analytics with no matching hits")
+    _remove_analytics_without_hits(ds)
+
+
+def _remove_analytics_without_hits(ds: HowlerDatastore):
+
+    matched_analytics = _find_analytics_with_hits(ds)
+
+    if matched_analytics is not None:
+        ds.analytic.delete_by_search_object(
+            {
+                "bool": {
+                    "filter": [
+                        {
+                            "bool": {
+                                "must_not": [
+                                    {"exists": {"field": "rule"}},
+                                    {"exists": {"field": "rule_type"}},
+                                ]
+                            }
+                        }
+                    ],
+                    "must_not": [{"terms": {"name": matched_analytics}}],
+                }
+            }
+        )
+        ds.analytic.commit()
+    else:
+        logger.warning(
+            "Aggregation search for matched analytics did not run or returned no results. "
+            "There is likely an issue with the query. Skipping cleanup."
+        )
+
+
+def _find_analytics_with_hits(ds: HowlerDatastore) -> list[str] | None:
+
+    total_analytics = ds.analytic.count("id:*", filters=None)["count"]
+
+    if total_analytics:
+        matched_analytics = ds.hit.search(
+            "howler.id:*",
+            aggregations=[
+                (
+                    "matched_analytics",
+                    {
+                        "terms": {
+                            "field": "howler.analytic",
+                            "size": total_analytics,
+                        }
+                    },
+                )
+            ],
+            rows=0,
+        )
+
+        if "matched_analytics" in matched_analytics["aggregations"]:
+            matched_analytic_names = [
+                bucket["key"] for bucket in matched_analytics["aggregations"]["matched_analytics"]["buckets"]
+            ]
+        else:
+            return None
+
+    else:
+        matched_analytic_names = []
+
+    return matched_analytic_names
+
+
+def setup_job(sched: BaseScheduler):
+    """Initialize the retention job"""
+    if not config.system.retention.enabled:
+        if not DEBUG or config.system.type == "production":
+            logger.warning("Retention cronjob disabled! This is not recommended for a production settings.")
+
+        return
+
+    logger.debug("Initializing retention cronjob with cron %s", config.system.retention.crontab)
+
+    if DEBUG:
+        _kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
+    else:
+        _kwargs = {}
+
+    if sched.get_job("retention"):
+        logger.debug("Retention job already running!")
+        return
+
+    sched.add_job(
+        id="retention",
+        func=execute,
+        trigger=CronTrigger.from_crontab(
+            config.system.retention.crontab, timezone=timezone(os.getenv("SCHEDULER_TZ", "America/Toronto"))
+        ),
+        **_kwargs,
+    )
+    logger.debug("Initialization complete")