howler-api 4.0.0.dev1207__tar.gz → 4.0.0.dev1246__tar.gz

{howler_api-4.0.0.dev1207 → howler_api-4.0.0.dev1246}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1207
+Version: 4.0.0.dev1246
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev1207 → howler_api-4.0.0.dev1246}/howler/cronjobs/retention.py

@@ -31,12 +31,74 @@ def execute():
 
     logger.debug("Deletion complete")
 
+    _execute_rules(ds)
+
     logger.debug("Cleaning analytics with no matching hits")
     _remove_analytics_without_hits(ds)
 
 
-def _remove_analytics_without_hits(ds: HowlerDatastore):
+def _execute_rules(ds: HowlerDatastore) -> None:
+    """Execute dynamic retention rules from config.
+
+    Iterates each enabled rule, computes its cutoff date, and deletes
+    hits that match the rule's query and are older than the cutoff.
+    Errors for individual rules are logged and skipped so that
+    remaining rules still execute. Emits a structured result log
+    for each rule execution.
+
+    Note: Rules are independent. If multiple rules match the same hit,
+    the hit is deleted by whichever rule runs first; subsequent matching
+    rules will no-op against the already-deleted document.
+
+    Args:
+        ds: Active HowlerDatastore instance.
+    """
+    rules = config.system.retention.rules
+    if not rules:
+        return
+
+    logger.info("Processing dynamic retention rules:")
+
+    for rule in rules:
+        result = {}
+
+        if not rule.enabled:
+            logger.debug("Skipping disabled retention rule '%s'", rule.name)
+            result = {"rule": rule.name, "status": "skipped", "reason": "disabled"}
+            logger.info("Rule result: '%s'", result)
+            continue
+
+        delta_kwargs = {str(rule.limit_unit): rule.limit_amount}
+        cutoff_dt = datetime.now(tz=timezone("UTC")) - timedelta(**delta_kwargs)
+        cutoff = cutoff_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+        combined_query = f"({rule.query}) AND event.created:{{* TO {cutoff}}}"
 
+        logger.debug(
+            "Executing retention rule '%s': query=%s cutoff=%s",
+            rule.name,
+            rule.query,
+            cutoff,
+        )
+
+        try:
+            count = ds.hit.count(combined_query, filters=None).get("count", -1)
+            ds.hit.delete_by_query(combined_query)
+            ds.hit.commit()
+            logger.debug("Retention rule '%s' complete", rule.name)
+            result = {"rule": rule.name, "status": "ok", "deleted": count, "cutoff": cutoff}
+        except Exception:
+            logger.error(
+                "Retention rule '%s' failed — skipping. query=%s",
+                rule.name,
+                combined_query,
+                exc_info=True,
+            )
+            result = {"rule": rule.name, "status": "error", "query": combined_query}
+
+        logger.info("Rule result: '%s'", result)
+
+
+def _remove_analytics_without_hits(ds: HowlerDatastore):
     matched_analytics = _find_analytics_with_hits(ds)
 
     if matched_analytics is not None:
@@ -66,7 +128,6 @@ def _remove_analytics_without_hits(ds: HowlerDatastore):
 
 
 def _find_analytics_with_hits(ds: HowlerDatastore) -> list[str] | None:
-
     total_analytics = ds.analytic.count("id:*", filters=None)["count"]
 
     if total_analytics:
@@ -127,3 +188,7 @@ def setup_job(sched: BaseScheduler):
         **_kwargs,
     )
     logger.debug("Initialization complete")
+
+
+if __name__ == "__main__":
+    execute()

{howler_api-4.0.0.dev1207 → howler_api-4.0.0.dev1246}/howler/odm/models/config.py

@@ -356,6 +356,26 @@ class Auth(BaseModel):
     oauth: OAuth = OAuth()
 
 
+class RetentionRule(BaseModel):
+    """A dynamic, query-scoped retention rule.
+
+    Pairs a Lucene query with a retention window. During the retention
+    cronjob run, hits matching the query that are older than the
+    specified window will be deleted.
+    """
+
+    name: str = Field(description="A human-readable label for this rule (used in logs).")
+    enabled: bool = Field(default=True, description="Whether this rule is active.")
+    query: str = Field(description="Lucene query that scopes which hits this rule applies to.")
+    limit_unit: Literal["days", "seconds", "microseconds", "milliseconds", "minutes", "hours", "weeks"] = Field(
+        default="days",
+        description="The unit to use when computing the retention limit for this rule.",
+    )
+    limit_amount: int = Field(
+        description="The number of limit_units to use when computing the retention limit for this rule.",
+    )
+
+
 class Retention(BaseModel):
     """Hit retention policy configuration.
 
@@ -382,6 +402,13 @@ class Retention(BaseModel):
         default="0 0 * * *",
         description="The crontab that denotes how often to run the retention job",
     )
+    rules: list[RetentionRule] = Field(
+        default=[],
+        description=(
+            "Optional list of dynamic retention rules. Each rule pairs a Lucene query "
+            "with a retention window. Rules run after the global sweep."
+        ),
+    )
 
 
 class ViewCleanup(BaseModel):

{howler_api-4.0.0.dev1207 → howler_api-4.0.0.dev1246}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev1207"
+version = "4.0.0.dev1246"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",