howler-api 2.12.0.dev285__py3-none-any.whl → 2.12.0.dev294__py3-none-any.whl

howler/cronjobs/view_cleanup.py

@@ -0,0 +1,88 @@
+import os
+from datetime import datetime
+from typing import Any, List
+
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.triggers.cron import CronTrigger
+from pytz import timezone
+
+from howler.common.logging import get_logger
+from howler.config import DEBUG, config
+
+logger = get_logger(__file__)
+
+
+def execute():
+    """Delete any pinned views that no longer exist"""
+    from howler.common.loader import datastore
+
+    # Initialize datastore
+    ds = datastore()
+    # fetch the first result from user ds (needed to initialize total)
+    result = ds.user.search("*:*", rows=250, fl="*")
+    total_user_count = result["total"]
+    user_list: List[Any] = result["items"]
+    # Do the same thing for the views
+    result = ds.view.search("*:*", rows=250)
+    total_view_count = result["total"]
+    view_list: List[Any] = result["items"]
+    view_ids: List[str] = []
+
+    # Collect all views
+    while len(view_list) < total_view_count:
+        view_list.extend(ds.view.search("*:*", rows=250, offset=len(user_list)))
+
+    # Collect all users
+    while len(user_list) < total_user_count:
+        user_list.extend(ds.user.search("*:*", rows=250, offset=len(user_list)))
+
+    for view in view_list:
+        view_ids.append(view["view_id"])
+
+    # Iterate over each user to see if the dashboard contains invalid entries (deleted views)
+    for user in user_list:
+        valid_entries = []
+        # No views/analytics saved to the dashboard? Skip it
+        if user["dashboard"] == []:
+            continue
+        for dashboard_entry in user["dashboard"]:
+            if dashboard_entry["type"] != "view" or (
+                dashboard_entry["type"] == "view" and dashboard_entry["entry_id"] in view_ids
+            ):
+                valid_entries.append(dashboard_entry)
+        # If the length of valid entries is less than the current dashboard, one or more pins are invalid
+        if len(valid_entries) < len(user["dashboard"]):
+            # set the user dashboard to valid entries
+            user["dashboard"] = valid_entries
+            # update the user
+            ds.user.save(user["uname"], user)
+
+
+def setup_job(sched: BaseScheduler):
+    """Initialize the view cleanup job"""
+    if not config.system.view_cleanup.enabled:
+        if not DEBUG or config.system.type == "production":
+            logger.warning("view cleanup cronjob disabled! This is not recommended for a production settings.")
+
+        return
+
+    logger.debug(f"Initializing view cleanup cronjob with cron {config.system.view_cleanup.crontab}")
+
+    if DEBUG:
+        _kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
+    else:
+        _kwargs = {}
+
+    if sched.get_job("view_cleanup"):
+        logger.debug("view cleanup job already running!")
+        return
+
+    sched.add_job(
+        id="view_cleanup",
+        func=execute,
+        trigger=CronTrigger.from_crontab(
+            config.system.view_cleanup.crontab, timezone=timezone(os.getenv("SCHEDULER_TZ", "America/Toronto"))
+        ),
+        **_kwargs,
+    )
+    logger.debug("Initialization complete")

howler/odm/models/config.py

@@ -250,10 +250,26 @@ class Retention(BaseModel):
     )
 
 
+class ViewCleanup(BaseModel):
+    enabled: bool = Field(
+        default=True,
+        description=(
+            "Whether to enable the view cleanup. If enabled, views pinned "
+            "to the dashboard that no longer exist in the backend will be cleared."
+        ),
+    )
+    crontab: str = Field(
+        default="0 0 * * *",
+        description="The crontab that denotes how often to run the view_cleanup job",
+    )
+
+
 class System(BaseModel):
     type: Literal["production", "staging", "development"] = Field(default="development", description="Type of system")
     retention: Retention = Retention()
     "Retention Configuration"
+    view_cleanup: ViewCleanup = ViewCleanup()
+    "View Cleanup Configuration"
 
 
 class UI(BaseModel):

{howler_api-2.12.0.dev285.dist-info → howler_api-2.12.0.dev294.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 2.12.0.dev285
+Version: 2.12.0.dev294
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-2.12.0.dev285.dist-info → howler_api-2.12.0.dev294.dist-info}/RECORD

@@ -51,6 +51,7 @@ howler/config.py,sha256=amwf4FXJ1ZKeooHqPlC_F691tg6Yaj4osKn_ExiUSS0,2128
 howler/cronjobs/__init__.py,sha256=GEhNsxPGATumlroMa-g6z5Dt9yy0QT93s3uuNC-GwIM,909
 howler/cronjobs/retention.py,sha256=I-vXXFo2W5eHkJ-xQuywU2-KfM9xqgii9Y99Qd9E-Vk,1777
 howler/cronjobs/rules.py,sha256=ZK3FmLsEF34-3vQzXkc4J6Y0P5-JSawMSnAkH9o-PeE,10406
+howler/cronjobs/view_cleanup.py,sha256=ULWLR1uFcRemRgkEDrMqmBamHiE0dgDvu49VK0qDkuw,3076
 howler/datastore/README.md,sha256=ekWl1YJSrHlZpU5PgBkPEzPWjdbTdav6Rd2P0ccIesw,4758
 howler/datastore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 howler/datastore/bulk.py,sha256=VfolZfiaBD4ZTK3j6IVVVq4GMjVXb5elrsGwwM_nONE,2829
@@ -96,7 +97,7 @@ howler/odm/models/assemblyline.py,sha256=_wcTiX3A6bhA2SGlK9tDF0v-uwLpIabXE8j2Fw9
 howler/odm/models/aws.py,sha256=pJVadJqubdgT27riCfp7bEKVP4XsMZB0ZUnKAbmCMd0,895
 howler/odm/models/azure.py,sha256=o7MZMMo9jh1SB8xXCajl_YSKP2nnnWsjx_DPT6LnQKg,710
 howler/odm/models/cbs.py,sha256=onUiJOGUxK3iy_-4XkGGwHxFiFq9Td_p59Kum4XaR-w,1366
-howler/odm/models/config.py,sha256=jviwXPvBLkOxpDzOYeVQ72-rl4jhlamcmvBrqtAJM-Y,16123
+howler/odm/models/config.py,sha256=WBjMuqWY7XYt4Gx7uWFpEJz0760yhr91Iw8bQZTBas4,16629
 howler/odm/models/dossier.py,sha256=Ob2qROrG2-DYzmVo2XVe4NJ8HjWGCoRAu2gPo6p9XGU,1244
 howler/odm/models/ecs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 howler/odm/models/ecs/agent.py,sha256=idSooyFCLuQAB7_RyEWTYW4-x9w5a3wpy2ct_-EDRQs,713
@@ -193,7 +194,7 @@ howler/utils/path.py,sha256=DfOU4i4zSs4wchHoE8iE7aWVLkTxiC_JRGepF2hBYBk,690
 howler/utils/socket_utils.py,sha256=nz1SklC9xBHUSfHyTJjpq3mbozX1GDf01WzdGxfaUII,2212
 howler/utils/str_utils.py,sha256=HE8Hqh2HlOLaj16w0H9zKOyDJLp-f1LQ50y_WeGZaEk,8389
 howler/utils/uid.py,sha256=p9dsqyvZ-lpiAuzZWCPCeEM99kdk0Ly9czf04HNdSuw,1341
-howler_api-2.12.0.dev285.dist-info/METADATA,sha256=8HJ5Mbd2tySG-zjH-z6_5UF1XyOABST62_OTju_kMyM,2815
-howler_api-2.12.0.dev285.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-howler_api-2.12.0.dev285.dist-info/entry_points.txt,sha256=Lu9SBGvwe0wczJHmc-RudC24lmQk7tv3ZBXon9RIihg,259
-howler_api-2.12.0.dev285.dist-info/RECORD,,
+howler_api-2.12.0.dev294.dist-info/METADATA,sha256=IyUU6T7TekQltonOSedkzSQ1mpcRu8uOkasVlXMWFuA,2815
+howler_api-2.12.0.dev294.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+howler_api-2.12.0.dev294.dist-info/entry_points.txt,sha256=Lu9SBGvwe0wczJHmc-RudC24lmQk7tv3ZBXon9RIihg,259
+howler_api-2.12.0.dev294.dist-info/RECORD,,