howler-api 2.13.0.dev329__py3-none-any.whl

howler/cronjobs/rules.py

@@ -0,0 +1,274 @@
+import hashlib
+import json
+import os
+import random
+import re
+import sys
+from datetime import datetime
+from typing import Any, Optional
+
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.triggers.cron import CronTrigger
+from pytz import timezone
+from sigma.backends.elasticsearch import LuceneBackend
+from sigma.rule import SigmaRule
+from yaml.scanner import ScannerError
+
+from howler.common.exceptions import HowlerValueError
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.config import DEBUG, HWL_ENABLE_RULES
+from howler.datastore.collection import ESCollection
+from howler.datastore.operations import OdmHelper, OdmUpdateOperation
+from howler.odm.models.analytic import Analytic
+from howler.odm.models.hit import Hit
+from howler.odm.models.howler_data import HitOperationType
+
+logger = get_logger(__file__)
+hit_helper = OdmHelper(Hit)
+
+__scheduler_instance: Optional[BaseScheduler] = None
+
+
+def create_correlated_bundle(rule: Analytic, query: str, correlated_hits: list[Hit]):
+    "Create a bundle based on the results of an analytic"
+    # We'll create a hash using the hashes of the children, and the analytic ID/current time
+    bundle_hash = hashlib.sha256()
+    bundle_hash.update(rule.analytic_id.encode())
+    bundle_hash.update(query.replace("now", datetime.now().isoformat()).encode())
+    for match in correlated_hits:
+        bundle_hash.update(match.howler.hash.encode())
+
+    hashed = bundle_hash.hexdigest()
+
+    # If a matching bundle exists already, just reused it (likely only ever lucene specific)
+    existing_result = datastore().hit.search(f"howler.hash:{hashed}", rows=1)
+    if existing_result["total"] > 0:
+        logger.debug(f"Rule hash {hashed} exists - skipping create")
+        return existing_result["items"][0]
+
+    child_ids = [match.howler.id for match in correlated_hits]
+
+    correlated_bundle = Hit(
+        {
+            "howler.analytic": rule.name,
+            "howler.detection": "Rule",
+            "howler.score": 0.0,
+            "howler.hash": hashed,
+            "howler.is_bundle": True,
+            "howler.hits": child_ids,
+            "howler.data": [
+                json.dumps(
+                    {
+                        "raw": rule.rule,
+                        "sanitized": query,
+                    }
+                )
+            ],
+            "event.created": "NOW",
+            "event.kind": "alert",
+            "event.module": rule.rule_type,
+            "event.provider": "howler",
+            "event.reason": f"Children match {query}",
+            "event.type": ["info"],
+        }
+    )
+    correlated_bundle.event.id = correlated_bundle.howler.id
+
+    datastore().hit.save(correlated_bundle.howler.id, correlated_bundle)
+
+    if len(child_ids) > 0:
+        datastore().hit.update_by_query(
+            f"howler.id:({' OR '.join(child_ids)})",
+            [
+                hit_helper.list_add(
+                    "howler.bundles",
+                    correlated_bundle.howler.id,
+                    if_missing=True,
+                ),
+                OdmUpdateOperation(
+                    ESCollection.UPDATE_APPEND,
+                    "howler.log",
+                    {
+                        "timestamp": "NOW",
+                        "key": "howler.bundles",
+                        "explanation": f"This hit was correlated by the analytic '{rule.name}'.",
+                        "new_value": correlated_bundle.howler.id,
+                        "previous_value": "None",
+                        "type": HitOperationType.APPENDED,
+                        "user": "Howler",
+                    },
+                ),
+            ],
+        )
+
+    return correlated_bundle
+
+
+def create_executor(rule: Analytic):  # noqa: C901
+    "Create a cronjob for a given analytic"
+
+    def execute():  # noqa: C901
+        "Execute the rule"
+        try:
+            if not rule.rule or not rule.rule_type:
+                logger.error("Invalid rule %s! Skipping", rule.analytic_id)
+                return
+
+            logger.info(
+                "Executing rule %s (%s)",
+                rule.name,
+                rule.analytic_id,
+            )
+
+            correlated_hits: Optional[list[Hit]] = None
+
+            if rule.rule_type in ["lucene", "sigma"]:
+                if rule.rule_type == "lucene":
+                    query = re.sub(r"\n+", " ", re.sub(r"#.+", "", rule.rule)).strip()
+                else:
+                    try:
+                        sigma_rule = SigmaRule.from_yaml(rule.rule)
+                    except ScannerError as e:
+                        raise HowlerValueError(
+                            f"Error when parsing yaml: {e.problem} {e.problem_mark}",
+                            cause=e,
+                        )
+
+                    es_collection = datastore().hit
+                    lucene_queries = LuceneBackend(index_names=[es_collection.index_name]).convert_rule(sigma_rule)
+
+                    query = " AND ".join([f"({q})" for q in lucene_queries])
+
+                num_hits = datastore().hit.search(query, rows=1)["total"]
+                if num_hits > 0:
+                    bundle = create_correlated_bundle(rule, query, [])
+                    datastore().hit.update_by_query(
+                        f"({query}) AND -howler.bundles:{bundle.howler.id}",
+                        [
+                            hit_helper.list_add(
+                                "howler.bundles",
+                                bundle.howler.id,
+                                if_missing=True,
+                            ),
+                            OdmUpdateOperation(
+                                ESCollection.UPDATE_APPEND,
+                                "howler.log",
+                                {
+                                    "timestamp": "NOW",
+                                    "key": "howler.bundles",
+                                    "explanation": f"This hit was correlated by the analytic '{rule.name}'.",
+                                    "new_value": bundle.howler.id,
+                                    "previous_value": "None",
+                                    "type": HitOperationType.APPENDED,
+                                    "user": "Howler",
+                                },
+                            ),
+                        ],
+                    )
+
+                    datastore().hit.commit()
+
+                    child_hits: list[Hit] = datastore().hit.search(
+                        f"howler.bundles:{bundle.howler.id}", rows=1000, fl="howler.id"
+                    )["items"]
+                    datastore().hit.update_by_query(
+                        f"howler.id:{bundle.howler.id}",
+                        [hit_helper.list_add("howler.hits", hit.howler.id, if_missing=True) for hit in child_hits],
+                    )
+
+            elif rule.rule_type == "eql":
+                query = rule.rule
+
+                result = datastore().hit.raw_eql_search(query, rows=25, fl=",".join(Hit.flat_fields().keys()))
+
+                if len(result["sequences"]) > 0:
+                    for sequence in result["sequences"]:
+                        if len(sequence) > 0:
+                            create_correlated_bundle(rule, query, sequence)
+
+                correlated_hits = result["items"]
+
+            else:  # pragma: no cover
+                raise HowlerValueError(f"Unknown rule type: {rule.rule_type}")  # noqa: TRY301
+
+            if correlated_hits and len(correlated_hits) > 0:
+                create_correlated_bundle(rule, query, correlated_hits)
+        except Exception as e:
+            logger.debug(e, exc_info=True)
+            if __scheduler_instance:
+                __scheduler_instance.remove_job(f"rule_{rule.analytic_id}")
+            # TODO: Allow restarting of rules
+            logger.critical(
+                f"Rule {rule.name} ({rule.analytic_id}) has been stopped, due to an exception: {type(e)}",
+                exc_info=True,
+            )
+
+    return execute
+
+
+def register_rules(new_rule: Optional[Analytic] = None, test_override: bool = False):
+    "Register all of the created analytic rules as cronjobs"
+    global __scheduler_instance
+    if not __scheduler_instance:  # pragma: no cover
+        logger.error("Scheduler instance does not exist!")
+        return
+
+    if "pytest" in sys.modules and not test_override:
+        logger.info("Skipping registration, running in a test environment")
+        return
+
+    if new_rule:
+        if __scheduler_instance.get_job(f"rule_{new_rule.analytic_id}"):
+            logger.info(f"Updating existing rule: {new_rule.analytic_id} on interval {new_rule.rule_crontab}")
+
+            # remove the existing job
+            __scheduler_instance.remove_job(f"rule_{new_rule.analytic_id}")
+        else:
+            logger.info(f"Registering new rule: {new_rule.analytic_id} on interval {new_rule.rule_crontab}")
+        rules = [new_rule]
+    else:
+        logger.debug("Registering rules")
+        rules: list[Analytic] = datastore().analytic.search("_exists_:rule")["items"]
+
+    total_initialized = 0
+    for rule in rules:
+        job_id = f"rule_{rule.analytic_id}"
+        interval = rule.rule_crontab or f"{random.randint(0, 59)} * * * *"  # noqa: S311
+
+        if __scheduler_instance.get_job(job_id):
+            logger.debug(f"Rule {job_id} already running!")
+            return
+
+        logger.debug(f"Initializing rule cronjob with:\tJob ID: {job_id}\tRule Name: {rule.name}\tCrontab: {interval}")
+
+        if DEBUG or new_rule:
+            _kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
+        else:
+            _kwargs = {}
+
+        total_initialized += 1
+        __scheduler_instance.add_job(
+            id=job_id,
+            func=create_executor(rule),
+            trigger=CronTrigger.from_crontab(interval, timezone=timezone(os.getenv("SCHEDULER_TZ", "America/Toronto"))),
+            **_kwargs,
+        )
+
+    logger.info(f"Initialized {total_initialized} rules")
+
+
+def setup_job(sched: BaseScheduler):
+    "Initialize the rules cronjobs"
+    if not DEBUG and not HWL_ENABLE_RULES:  # pragma: no cover
+        logger.debug("Rule integration disabled")
+        return
+
+    logger.debug("Rule integration enabled")
+
+    global __scheduler_instance
+    __scheduler_instance = sched
+
+    register_rules()
+
+    logger.debug("Initialization complete")

howler/cronjobs/view_cleanup.py

@@ -0,0 +1,88 @@
+import os
+from datetime import datetime
+from typing import Any, List
+
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.triggers.cron import CronTrigger
+from pytz import timezone
+
+from howler.common.logging import get_logger
+from howler.config import DEBUG, config
+
+logger = get_logger(__file__)
+
+
+def execute():
+    """Delete any pinned views that no longer exist"""
+    from howler.common.loader import datastore
+
+    # Initialize datastore
+    ds = datastore()
+    # fetch the first result from user ds (needed to initialize total)
+    result = ds.user.search("*:*", rows=250, fl="*")
+    total_user_count = result["total"]
+    user_list: List[Any] = result["items"]
+    # Do the same thing for the views
+    result = ds.view.search("*:*", rows=250)
+    total_view_count = result["total"]
+    view_list: List[Any] = result["items"]
+    view_ids: List[str] = []
+
+    # Collect all views
+    while len(view_list) < total_view_count:
+        view_list.extend(ds.view.search("*:*", rows=250, offset=len(user_list)))
+
+    # Collect all users
+    while len(user_list) < total_user_count:
+        user_list.extend(ds.user.search("*:*", rows=250, offset=len(user_list)))
+
+    for view in view_list:
+        view_ids.append(view["view_id"])
+
+    # Iterate over each user to see if the dashboard contains invalid entries (deleted views)
+    for user in user_list:
+        valid_entries = []
+        # No views/analytics saved to the dashboard? Skip it
+        if user["dashboard"] == []:
+            continue
+        for dashboard_entry in user["dashboard"]:
+            if dashboard_entry["type"] != "view" or (
+                dashboard_entry["type"] == "view" and dashboard_entry["entry_id"] in view_ids
+            ):
+                valid_entries.append(dashboard_entry)
+        # If the length of valid entries is less than the current dashboard, one or more pins are invalid
+        if len(valid_entries) < len(user["dashboard"]):
+            # set the user dashboard to valid entries
+            user["dashboard"] = valid_entries
+            # update the user
+            ds.user.save(user["uname"], user)
+
+
+def setup_job(sched: BaseScheduler):
+    """Initialize the view cleanup job"""
+    if not config.system.view_cleanup.enabled:
+        if not DEBUG or config.system.type == "production":
+            logger.warning("view cleanup cronjob disabled! This is not recommended for a production settings.")
+
+        return
+
+    logger.debug(f"Initializing view cleanup cronjob with cron {config.system.view_cleanup.crontab}")
+
+    if DEBUG:
+        _kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
+    else:
+        _kwargs = {}
+
+    if sched.get_job("view_cleanup"):
+        logger.debug("view cleanup job already running!")
+        return
+
+    sched.add_job(
+        id="view_cleanup",
+        func=execute,
+        trigger=CronTrigger.from_crontab(
+            config.system.view_cleanup.crontab, timezone=timezone(os.getenv("SCHEDULER_TZ", "America/Toronto"))
+        ),
+        **_kwargs,
+    )
+    logger.debug("Initialization complete")

howler/datastore/README.md

@@ -0,0 +1,112 @@
+# Elasticsearch datastore support
+
+This component aims to simplify the connection between your app and Elasticsearch by providing a single interface to use with all your different indices.
+
+Advantages:
+
+- Connection keep alive and retries
+  - No need to worry if you elastic cluster goes down, your app will resume where it was when it's back online.
+- Keep index management simple:
+  - If you register a new index to the data, the associated index in Elastic will be created.
+  - If you add or remove a field in an index, the associated index in Elastic will be updated.
+  - You can easily re-index, re-shard or change and index replication.
+- Support bulk operations and archiving
+- Support all basic operation get, put, update, search, facet, stats, histogram...
+
+Disadvantages:
+
+- Search uses lucene only (covers 99% of use-cases but may be extended if needed)
+
+## Naming convention
+
+Take note of the different naming convention:
+
+- An Elastic index will be refered as a `Collection` because it may have multiple indexes as it's backend
+- The object that holds multiple collection as a `Datastore`
+
+## Usage
+
+### Instanciating a datastore
+
+When instanciating an datastore object, there are no collection associated to it. You need to register each collection in the object so it be kept in sync and have access to it. After the collection is registered, you have access to this collection as a property of the datastore object.
+
+Example:
+
+```python
+from howler.common import loader
+from myapp.models.mymodel import MyModel
+
+ds = loader.get_esstore()
+ds.register('mymodel', MyModel)
+
+my_document = ds.mymodel.get(document_id)
+
+```
+
+### Creating your own datastore
+
+This get very complicated when you have multiple collections which is why we recommend that you create your own datastore helper class that has all collections pre-loaded.
+
+Example:
+
+```python
+from howler.common import loader
+from howler.datastore.collection import ESCollection
+from howler.datastore.store import ESStore
+
+from myapp.models.mycollection import MyCollection
+# ... + all other collection
+
+
+class MyDatastore(object):
+    def __init__(self, esstore_object: ESStore = None ):
+
+        self.ds = esstore_object or loader.get_esstore()
+        self.ds.register('mycollection', MyCollection)
+        # ... + all other collections
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.ds.close()
+
+    @property
+    def mycollection(self) -> ESCollection[MyCollection]:
+        return self.ds.mycollection
+
+    # ... + all other properties tied to the different collections
+```
+
+### Collection functions
+
+Once you've setup your own datastore object, you can start using the different functions that each collection offers. Here's a breakdown:
+
+- `archive(query)`: Send all meatching documents to the archive of the collection
+- `multiget(id_list)`: Get multiple documents for the id_list
+- `exists(id)`: Check if a document matching this id exists
+- `get(id)`: Get a document matching the id (retry twice if missing)
+- `get_if_exists(id)`: Get a document matching the id (do not retry)
+- `require(id)`: Try to get a document matching the id and retry forever until it exists
+- `save(id, doc)`: Save a document to this id and overrite it if it exists
+- `delete(id)`: Delete the document matching this id
+- `delete_by_query(query)`: Delete all documents matching this query
+- `update(id, operations)`: Perform the following update operation on this id
+- `update_by_query(query, operations)`: Perform the following update operation all document matching this query
+- `search(query)`: Find document matching the query and return one page
+- `stream_search(query)`: Return all document matching the query
+- `histogram(field, start, end, gap)`: Count how many documents are found in each gap from the start to the end (works on dates and int fields)
+- `facet(field)`: Return the top 10 values of a field
+- `stats(field)`: Generate min, max, avg, count of an int field
+- `grouped_search(group_field, query)`: Find all document matching a query and group the result by this field
+- `fields()`: List all fields of a collection
+
+Management related function: (*These should not really be used in normal code but are more tailored to fix issues and test the system*)
+
+- `commit()`: Save the indexes to disc now and make all documents available for search
+- `keys()`: Return ids of all the document in the index
+- `fix_ilm()`: Fix Index Lifecycle management configuration for the associated indices
+- `fix_replicas()`: Fix the number of copies of the associated indices
+- `fix_shards()`: Fix the number of shards of the associated indices
+- `reindex()`: Reindex all documents
+- `wipe()`: Delete and create empty version of this collection

howler/datastore/bulk.py

@@ -0,0 +1,72 @@
+import json
+from copy import deepcopy
+
+
+class ElasticBulkPlan(object):
+    def __init__(self, indexes, model=None):
+        self.indexes = indexes
+        self.model = model
+        self.operations = []
+
+    @property
+    def empty(self):
+        return len(self.operations) == 0
+
+    def add_delete_operation(self, doc_id, index=None):
+        if index:
+            self.operations.append(json.dumps({"delete": {"_index": index, "_id": doc_id}}))
+        else:
+            for cur_index in self.indexes:
+                self.operations.append(json.dumps({"delete": {"_index": cur_index, "_id": doc_id}}))
+
+    def add_insert_operation(self, doc_id, doc, index=None):
+        if isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(json.dumps({"create": {"_index": index or self.indexes[0], "_id": doc_id}}))
+        self.operations.append(json.dumps(saved_doc))
+
+    def add_upsert_operation(self, doc_id, doc, index=None):
+        if isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+        saved_doc["id"] = doc_id
+
+        self.operations.append(json.dumps({"update": {"_index": index or self.indexes[0], "_id": doc_id}}))
+        self.operations.append(json.dumps({"doc": saved_doc, "doc_as_upsert": True}))
+
+    def add_update_operation(self, doc_id, doc, index=None):
+        if isinstance(doc, self.model):
+            saved_doc = doc.as_primitives(hidden_fields=True)
+        elif self.model:
+            saved_doc = self.model(doc, mask=list(doc.keys())).as_primitives(hidden_fields=True)
+        else:
+            if not isinstance(doc, dict):
+                saved_doc = {"__non_doc_raw__": doc}
+            else:
+                saved_doc = deepcopy(doc)
+
+        if index:
+            self.operations.append(json.dumps({"update": {"_index": index, "_id": doc_id}}))
+            self.operations.append(json.dumps({"doc": saved_doc}))
+        else:
+            for cur_index in self.indexes:
+                self.operations.append(json.dumps({"update": {"_index": cur_index, "_id": doc_id}}))
+                self.operations.append(json.dumps({"doc": saved_doc}))
+
+    def get_plan_data(self):
+        return "\n".join(self.operations)