howler-api 3.4.0.dev927__tar.gz → 3.4.0.dev929__tar.gz

{howler_api-3.4.0.dev927 → howler_api-3.4.0.dev929}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 3.4.0.dev927
+Version: 3.4.0.dev929
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-3.4.0.dev927 → howler_api-3.4.0.dev929}/howler/api/v1/search.py

@@ -78,6 +78,7 @@ def search(index, **kwargs):
     sort                =>   How to sort the results (not available in deep paging)
     fl                  =>   List of fields to return
     timeout             =>   Maximum execution time (ms)
+    use_archive         =>   Allow access to the datastore achive (Default: False)
     track_total_hits    =>   Track the total number of query matches, instead of stopping at 10000 (Default: False)
     metadata            =>   A list of additional features to be added to the result alongside the raw results
 
@@ -117,9 +118,18 @@ def search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters", "metadata"]
+    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
+    params.update(
+        {
+            k: str(req_data.get(k, "false")).lower() in ["true", ""]
+            for k in boolean_fields
+            if req_data.get(k, None) is not None
+        }
+    )
+
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -340,9 +350,18 @@ def sigma_search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters"]
+    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
+    params.update(
+        {
+            k: str(req_data.get(k, "false")).lower() in ["true", ""]
+            for k in boolean_fields
+            if req_data.get(k, None) is not None
+        }
+    )
+
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -501,6 +520,7 @@ def count(index, **kwargs):
     Optional Arguments:
     filters             =>   List of additional filter queries limit the data
     timeout             =>   Maximum execution time (ms)
+    use_archive         =>   Allow access to the datastore achive (Default: False)
 
     Data Block:
     # Note that the data block is for POST requests only!
@@ -523,6 +543,15 @@ def count(index, **kwargs):
 
     params, req_data = generate_params(request, [], [])
 
+    boolean_fields = ["use_archive"]
+    params.update(
+        {
+            k: str(req_data.get(k, "false")).lower() in ["true", ""]
+            for k in boolean_fields
+            if req_data.get(k, None) is not None
+        }
+    )
+
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 

howler_api-3.4.0.dev929/howler/cronjobs/retention.py

@@ -0,0 +1,129 @@
+import os
+from datetime import datetime, timedelta
+from typing import Any
+
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.triggers.cron import CronTrigger
+from pytz import timezone
+
+from howler.common.logging import get_logger
+from howler.config import DEBUG, config
+from howler.datastore.howler_store import HowlerDatastore
+
+logger = get_logger(__file__)
+
+
+def execute():
+    """Delete any hits older than the configured time"""
+    from howler.common.loader import datastore
+
+    delta_kwargs = {str(config.system.retention.limit_unit): config.system.retention.limit_amount}
+
+    cutoff = (datetime.now() - timedelta(**delta_kwargs)).strftime("%Y-%m-%d")
+
+    logger.debug("Removing hits older than %s", cutoff)
+
+    ds = datastore()
+
+    ds.hit.delete_by_query(f"event.created:{{* TO {cutoff}}} OR howler.expiry:{{* TO now}}")
+
+    ds.hit.commit()
+
+    logger.debug("Deletion complete")
+
+    logger.debug("Cleaning analytics with no matching hits")
+    _remove_analytics_without_hits(ds)
+
+
+def _remove_analytics_without_hits(ds: HowlerDatastore):
+
+    matched_analytics = _find_analytics_with_hits(ds)
+
+    if matched_analytics is not None:
+        ds.analytic.delete_by_search_object(
+            {
+                "bool": {
+                    "filter": [
+                        {
+                            "bool": {
+                                "must_not": [
+                                    {"exists": {"field": "rule"}},
+                                    {"exists": {"field": "rule_type"}},
+                                ]
+                            }
+                        }
+                    ],
+                    "must_not": [{"terms": {"name": matched_analytics}}],
+                }
+            }
+        )
+        ds.analytic.commit()
+    else:
+        logger.warning(
+            "Aggregation search for matched analytics did not run or returned no results. "
+            "There is likely an issue with the query. Skipping cleanup."
+        )
+
+
+def _find_analytics_with_hits(ds: HowlerDatastore) -> list[str] | None:
+
+    total_analytics = ds.analytic.count("id:*", filters=None)["count"]
+
+    if total_analytics:
+        matched_analytics = ds.hit.search(
+            "howler.id:*",
+            aggregations=[
+                (
+                    "matched_analytics",
+                    {
+                        "terms": {
+                            "field": "howler.analytic",
+                            "size": total_analytics,
+                        }
+                    },
+                )
+            ],
+            rows=0,
+        )
+
+        if "matched_analytics" in matched_analytics["aggregations"]:
+            matched_analytic_names = [
+                bucket["key"] for bucket in matched_analytics["aggregations"]["matched_analytics"]["buckets"]
+            ]
+        else:
+            return None
+
+    else:
+        matched_analytic_names = []
+
+    return matched_analytic_names
+
+
+def setup_job(sched: BaseScheduler):
+    """Initialize the retention job"""
+    if not config.system.retention.enabled:
+        if not DEBUG or config.system.type == "production":
+            logger.warning("Retention cronjob disabled! This is not recommended for a production settings.")
+
+        return
+
+    logger.debug("Initializing retention cronjob with cron %s", config.system.retention.crontab)
+
+    if DEBUG:
+        _kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
+    else:
+        _kwargs = {}
+
+    if sched.get_job("retention"):
+        logger.debug("Retention job already running!")
+        return
+
+    sched.add_job(
+        id="retention",
+        func=execute,
+        trigger=CronTrigger.from_crontab(
+            config.system.retention.crontab, timezone=timezone(os.getenv("SCHEDULER_TZ", "America/Toronto"))
+        ),
+        **_kwargs,
+    )
+    logger.debug("Initialization complete")