howler-api 2.12.0.dev285__tar.gz → 2.12.0.dev286__tar.gz

{howler_api-2.12.0.dev285 → howler_api-2.12.0.dev286}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 2.12.0.dev285
+Version: 2.12.0.dev286
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

howler_api-2.12.0.dev286/howler/cronjobs/view_cleanup.py

@@ -0,0 +1,88 @@
+import os
+from datetime import datetime
+from typing import Any, List
+
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.triggers.cron import CronTrigger
+from pytz import timezone
+
+from howler.common.logging import get_logger
+from howler.config import DEBUG, config
+
+logger = get_logger(__file__)
+
+
+def execute():
+    """Delete any pinned views that no longer exist"""
+    from howler.common.loader import datastore
+
+    # Initialize datastore
+    ds = datastore()
+    # fetch the first result from user ds (needed to initialize total)
+    result = ds.user.search("*:*", rows=250, fl="*")
+    total_user_count = result["total"]
+    user_list: List[Any] = result["items"]
+    # Do the same thing for the views
+    result = ds.view.search("*:*", rows=250)
+    total_view_count = result["total"]
+    view_list: List[Any] = result["items"]
+    view_ids: List[str] = []
+
+    # Collect all views
+    while len(view_list) < total_view_count:
+        view_list.extend(ds.view.search("*:*", rows=250, offset=len(user_list)))
+
+    # Collect all users
+    while len(user_list) < total_user_count:
+        user_list.extend(ds.user.search("*:*", rows=250, offset=len(user_list)))
+
+    for view in view_list:
+        view_ids.append(view["view_id"])
+
+    # Iterate over each user to see if the dashboard contains invalid entries (deleted views)
+    for user in user_list:
+        valid_entries = []
+        # No views/analytics saved to the dashboard? Skip it
+        if user["dashboard"] == []:
+            continue
+        for dashboard_entry in user["dashboard"]:
+            if dashboard_entry["type"] != "view" or (
+                dashboard_entry["type"] == "view" and dashboard_entry["entry_id"] in view_ids
+            ):
+                valid_entries.append(dashboard_entry)
+        # If the length of valid entries is less than the current dashboard, one or more pins are invalid
+        if len(valid_entries) < len(user["dashboard"]):
+            # set the user dashboard to valid entries
+            user["dashboard"] = valid_entries
+            # update the user
+            ds.user.save(user["uname"], user)
+
+
+def setup_job(sched: BaseScheduler):
+    """Initialize the view cleanup job"""
+    if not config.system.view_cleanup.enabled:
+        if not DEBUG or config.system.type == "production":
+            logger.warning("view cleanup cronjob disabled! This is not recommended for a production settings.")
+
+        return
+
+    logger.debug(f"Initializing view cleanup cronjob with cron {config.system.view_cleanup.crontab}")
+
+    if DEBUG:
+        _kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
+    else:
+        _kwargs = {}
+
+    if sched.get_job("view_cleanup"):
+        logger.debug("view cleanup job already running!")
+        return
+
+    sched.add_job(
+        id="view_cleanup",
+        func=execute,
+        trigger=CronTrigger.from_crontab(
+            config.system.view_cleanup.crontab, timezone=timezone(os.getenv("SCHEDULER_TZ", "America/Toronto"))
+        ),
+        **_kwargs,
+    )
+    logger.debug("Initialization complete")

{howler_api-2.12.0.dev285 → howler_api-2.12.0.dev286}/howler/odm/models/config.py

@@ -250,10 +250,26 @@ class Retention(BaseModel):
     )
 
 
+class ViewCleanup(BaseModel):
+    enabled: bool = Field(
+        default=True,
+        description=(
+            "Whether to enable the view cleanup. If enabled, views pinned "
+            "to the dashboard that no longer exist in the backend will be cleared."
+        ),
+    )
+    crontab: str = Field(
+        default="0 0 * * *",
+        description="The crontab that denotes how often to run the view_cleanup job",
+    )
+
+
 class System(BaseModel):
     type: Literal["production", "staging", "development"] = Field(default="development", description="Type of system")
     retention: Retention = Retention()
     "Retention Configuration"
+    view_cleanup: ViewCleanup = ViewCleanup()
+    "View Cleanup Configuration"
 
 
 class UI(BaseModel):

{howler_api-2.12.0.dev285 → howler_api-2.12.0.dev286}/pyproject.toml

@@ -148,7 +148,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "2.12.0.dev285"
+version = "2.12.0.dev286"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",