howler-api 3.0.0.dev374__py3-none-any.whl

howler/external/generate_mitre.py

@@ -0,0 +1,96 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+import yaml
+
+
+def run(writepath):
+    """Run the script to generate mitre lookups"""
+    dirname = Path(writepath)
+    print(f"Generating mitre lookups to {dirname}")
+    file_name = os.path.join(os.getcwd(), "enterprise-attack.json")
+    tactics_name = dirname / "tactics.yml"
+    techniques_name = dirname / "techniques.yml"
+
+    if not os.path.exists(file_name):
+        print("Pulling mitre attack data")
+        response = requests.get(
+            "https://raw.githubusercontent.com/mitre-attack/attack-stix-data/"
+            + "master/enterprise-attack/enterprise-attack.json",
+            timeout=10,
+        )
+
+        response.raise_for_status()
+
+        with open(file_name, "w") as f:
+            f.write(response.text)
+
+        print("Done!")
+
+    print("Processing mitre attack data")
+    with open(file_name, "r") as f:
+        data = json.load(f)["objects"]
+
+        tactics_list = sorted(
+            [e for e in data if e["type"] == "x-mitre-tactic"],
+            key=lambda x: x["external_references"][0]["external_id"],
+        )
+
+        tactics_list = [
+            {
+                "name": e["name"],
+                "key": e["external_references"][0]["external_id"],
+                "url": e["external_references"][0]["url"],
+                # "description": e["description"].strip(),
+            }
+            for e in tactics_list
+        ]
+
+        tactics = {}
+        for tactic in tactics_list:
+            tactics[tactic["key"]] = tactic
+
+        print(f"Writing tactics to {tactics_name}")
+        with open(tactics_name, "w") as f:
+            yaml.dump(tactics, f)
+
+        techniques_list = sorted(
+            [e for e in data if e["type"] == "attack-pattern"],
+            key=lambda x: x["external_references"][0]["external_id"],
+        )
+
+        techniques_list = [
+            {
+                "name": e["name"],
+                "key": e["external_references"][0]["external_id"],
+                "url": e["external_references"][0]["url"],
+                # "description": e["description"].strip(),
+            }
+            for e in techniques_list
+        ]
+
+        techniques = {technique["key"]: technique for technique in techniques_list}
+
+        print(f"Writing techniques to {techniques_name}")
+        with open(techniques_name, "w") as f:
+            yaml.dump(techniques, f)
+
+    print("Done!")
+
+
+def main():
+    "Main run function"
+    writepath = None
+    try:
+        writepath = sys.argv[1]
+    except Exception:
+        writepath = "lookups"
+
+    run(writepath)
+
+
+if __name__ == "__main__":
+    main()

howler/external/generate_sigma_rules.py

@@ -0,0 +1,31 @@
+import shlex
+import shutil
+import subprocess
+from pathlib import Path
+from tempfile import mkdtemp
+
+
+def main():
+    """Run the script to generate sigma rule lookups"""
+    print("Generating sigma yaml")
+
+    git_dir = Path(mkdtemp())
+    subprocess.call(shlex.split((f"git clone git@github.com:SigmaHQ/sigma.git {git_dir} --depth 1")))
+
+    output_dir = Path(__file__).parent.parent / "odm" / "sigma"
+    if not output_dir.exists():
+        output_dir.mkdir()
+
+    print("Copying files")
+    for network_yaml in (git_dir / "rules" / "network").glob("**/*.yml"):
+        print(f"  {network_yaml.relative_to(((git_dir / 'rules' / 'network')))}")
+        new_file = output_dir / network_yaml.name
+        shutil.copyfile(network_yaml, new_file)
+
+    shutil.rmtree(git_dir)
+
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()

howler/external/generate_tlds.py

@@ -0,0 +1,47 @@
+import os
+
+import requests
+
+
+def get_tlds(url):
+    "Get a list of top-level domains"
+    comments = []
+    tlds = []
+
+    response = requests.get(url, timeout=10)
+    for line in response.text.splitlines():
+        if not line:
+            continue
+        if line.startswith("#"):
+            comments.append(line)
+        else:
+            tlds.append(line)
+
+    return comments, tlds
+
+
+if __name__ == "__main__":
+    tlds_url = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
+    tlds_location = "../howler/common/net_static.py"
+    if not os.path.exists(tlds_location):
+        print(
+            "Could not find net_static.py file. Make sure you run this script "
+            "in its home directory otherwise this won't work."
+        )
+        exit(1)
+
+    comments, tlds = get_tlds(tlds_url)
+    comments_lines = "\n".join(comments)
+    tlds_lines = '",\n    "'.join(tlds)
+
+    with open(tlds_location, "w") as tlds_fh:
+        tlds_fh.write(
+            "# This file is generated using generate_tlds.py script\n"
+            "# DO NOT EDIT! Re-run the script instead...\n\n"
+            f"# Top level domains from: {tlds_url}\n"
+            f"{comments_lines}\n"
+            f'TLDS_ALPHA_BY_DOMAIN = {{\n    "{tlds_lines}"\n}}\n'
+        )
+
+    print(f"TLDS list file written into: {tlds_location}")
+    print("You can now commit the new net_static.py file to your git.")

howler/external/reindex_data.py

@@ -0,0 +1,66 @@
+import json
+import sys
+import time
+from typing import Callable
+
+DELAY = 5
+
+
+if __name__ == "__main__":
+    print("This script will allow you to reindex all indexes in elasticsearch.")
+    print("For obvious reasons, be EXTREMELY CAREFUL running this code.")
+    print()
+    answer = input("Are you sure you want to reindex data for an index in this cluster? [yes/NO]\n")
+
+    if not answer.startswith("y"):
+        print("Confirmation not provided, stopping.")
+        sys.exit(1)
+
+    from howler.datastore.collection import ESCollection
+
+    ESCollection.IGNORE_ENSURE_COLLECTION = True
+
+    from howler.common import loader
+
+    ds = loader.datastore(archive_access=False)
+
+    indexes: dict[str, tuple[ESCollection, Callable]] = {
+        "analytic": (ds.analytic, ds.analytic.reindex),
+        "hit": (ds.hit, ds.hit.reindex),
+        "view": (ds.view, ds.view.reindex),
+        "template": (ds.template, ds.template.reindex),
+        "overview": (ds.overview, ds.overview.reindex),
+        "action": (ds.action, ds.action.reindex),
+        "user": (ds.user, ds.user.reindex),
+        "dossier": (ds.dossier, ds.dossier.reindex),
+    }
+
+    print("Which index will you reindex?")
+    index_answer = input(", ".join(indexes.keys()) + "\n> ")
+
+    if index_answer not in indexes:
+        print("Invalid index provided, stopping.")
+        sys.exit(1)
+
+    print("Index schema:")
+    print(json.dumps(indexes[index_answer][0]._get_index_mappings(), indent=2))
+
+    print("\nYou will be reindexing the following indexes:")
+    print("\n".join(indexes[index_answer][0].index_list_full))
+
+    answer = input(("\nAre you sure you want to reindex these indexes? [yes/NO]\n"))
+    print()
+
+    if not answer.startswith("y"):
+        print("Confirmation not provided, stopping.")
+        sys.exit(1)
+
+    for i in range(2 * DELAY):
+        print(f"Reindexing in {2 * DELAY - i}...", end="\r")
+        time.sleep(1)
+
+    print()
+
+    result = indexes[index_answer][1]()
+
+    print(f"Reindex complete. Success: {result}.")

howler/external/wipe_databases.py

@@ -0,0 +1,58 @@
+import sys
+import time
+
+DELAY = 5
+
+if __name__ == "__main__":
+    print("This script will allow you to completely remove all howler data and indexes from elasticsearch.")
+    print("For obvious reasons, be EXTREMELY CAREFUL running this code.")
+
+    for i in range(DELAY):
+        print(f"Continuing in {str(DELAY - i)}...", end="\r")
+        time.sleep(1)
+    print()
+    index = input("\nWhat index do you want to wipe? (Supported options are [hit, user]):\n")
+
+    if index not in ["hit", "user"]:
+        print("Invalid index.")
+        sys.exit(1)
+
+    answer = input(f"\nSelected index is {index}. Are you sure you want to wipe all data in this index? [yes/NO]\n")
+
+    if not answer.startswith("y"):
+        print("Confirmation not provided, stopping.")
+        sys.exit(1)
+
+    answer = input(
+        (
+            "\nSeriously, this will completely wipe the index. It'll go poof.\n"
+            "Are you sure you want to wipe all data in this index? [yes/NO]\n"
+        )
+    )
+    print()
+
+    if not answer.startswith("y"):
+        print("Confirmation not provided, stopping.")
+        sys.exit(1)
+
+    for i in range(2 * DELAY):
+        print(f"Deleting {index} in {2 * DELAY - i}...", end="\r")
+        time.sleep(1)
+
+    print()
+
+    from howler.datastore.collection import ESCollection
+
+    ESCollection.IGNORE_ENSURE_COLLECTION = True
+
+    from howler.common import loader
+    from howler.odm.random_data import wipe_hits, wipe_users
+
+    ds = loader.datastore(archive_access=False)
+
+    if index == "user":
+        wipe_users(ds)
+    else:
+        wipe_hits(ds)
+
+    print(f"Wiped {index}.")

howler/gunicorn_config.py

@@ -0,0 +1,25 @@
+import multiprocessing
+from os import environ as env
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Port to bind to
+bind = f":{int(env.get('PORT', 5000))}"
+
+# Number of processes to launch
+workers = int(env.get("WORKERS", multiprocessing.cpu_count()))
+
+# Number of concurrent handled connections
+threads = int(env.get("THREADS", 4))
+worker_connections = int(env.get("WORKER_CONNECTIONS", "1000"))
+
+# Recycle the process after X request randomized by the jitter
+max_requests = int(env.get("MAX_REQUESTS", "1000"))
+max_requests_jitter = int(env.get("MAX_REQUESTS_JITTER", "100"))
+
+# Connection timeouts
+graceful_timeout = int(env.get("GRACEFUL_TIMEOUT", "30"))
+# Official microsoft documentation suggest 600
+timeout = int(env.get("TIMEOUT", "360"))

howler/healthz.py

@@ -0,0 +1,47 @@
+from flask import Blueprint, abort, make_response
+
+from howler.common.loader import datastore
+
+API_PREFIX = "/api/healthz"
+healthz = Blueprint("healthz", __name__, url_prefix=API_PREFIX)
+
+
+@healthz.route("/live")
+def liveness(**_):
+    """Check if the API is live
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    OK or FAIL
+    """
+    return make_response("OK")
+
+
+@healthz.route("/ready")
+def readyness(**_):
+    """Check if the API is Ready
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    OK or FAIL
+    """
+    if datastore().ds.ping():
+        return make_response("OK")
+    else:
+        abort(503)
+
+
+@healthz.errorhandler(503)
+def error(_):
+    "Handle errors exposed in healthz routes"
+    return "FAIL", 503

howler/helper/azure.py

@@ -0,0 +1,50 @@
+import requests
+
+from howler.common.exceptions import HowlerException
+from howler.common.logging import get_logger
+from howler.config import config
+from howler.utils.str_utils import default_string_value
+
+logger = get_logger(__file__)
+
+
+def azure_obo(token: str) -> str:
+    """OBO an azure access token to MS Graph
+
+    Args:
+        token (str): The azure access token
+
+    Raises:
+        HowlerException: OBO failed
+
+    Returns:
+        str: The new access token with updated privileges
+    """
+    azure_provider_config = config.auth.oauth.providers["azure"]
+
+    logger.debug("OBOing to MS Graph")
+    # Azure is a special case here, as we need to OBO to MS Graph
+    data = {
+        "client_id": default_string_value(
+            azure_provider_config.client_id,
+            env_name="AZURE_CLIENT_ID",
+        ),
+        "scope": "https://graph.microsoft.com/user.read",
+        "assertion": token,
+        "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+        "client_secret": default_string_value(
+            azure_provider_config.client_secret,
+            env_name="AZURE_CLIENT_SECRET",
+        ),
+        "requested_token_use": "on_behalf_of",
+    }
+
+    if not azure_provider_config.access_token_url:
+        raise HowlerException("Azure OBO failed - access_token_url must be set!")
+
+    resp = requests.post(azure_provider_config.access_token_url, data=data, timeout=10)
+
+    if not resp.ok:
+        raise HowlerException(f"Azure OBO failed. Reason: {resp.content!r}")
+
+    return resp.json()["access_token"]

howler/helper/discover.py

@@ -0,0 +1,59 @@
+import sys
+import typing
+from typing import Optional
+
+import requests
+
+from howler.common.logging import get_logger
+from howler.config import config
+
+logger = get_logger(__file__)
+DISCO_CACHE = {}
+
+
+def get_apps_list(discovery_url: Optional[str]) -> list[dict[str, str]]:
+    """Get a list of apps from the discovery service
+
+    Returns:
+        list[dict[str, str]]: A list of other apps
+    """
+    if discovery_url not in DISCO_CACHE:
+        apps = []
+
+        if "pytest" in sys.modules:
+            logger.info("Skipping discovery, running in a test environment")
+
+        try:
+            resp = requests.get(
+                typing.cast(str, discovery_url or config.ui.discover_url),
+                headers={"accept": "application/json"},
+                timeout=5,
+            )
+            if resp.ok:
+                data = resp.json()
+                for app in data["applications"]["application"]:
+                    try:
+                        url = app["instance"][0]["hostName"]
+                        if "howler" not in url:
+                            apps.append(
+                                {
+                                    "alt": app["instance"][0]["metadata"]["alternateText"],
+                                    "name": app["name"],
+                                    "img_d": app["instance"][0]["metadata"]["imageDark"],
+                                    "img_l": app["instance"][0]["metadata"]["imageLight"],
+                                    "route": url,
+                                    "classification": app["instance"][0]["metadata"]["classification"],
+                                }
+                            )
+
+                    except Exception:
+                        logger.exception(f"Failed to parse get app: {str(app)}")
+            else:
+                logger.warning(f"Invalid response from server for apps discovery: {discovery_url}")
+        except Exception:
+            logger.exception(f"Failed to get apps from discover URL: {discovery_url}")
+
+        DISCO_CACHE[discovery_url] = sorted(apps, key=lambda k: ["name"])
+        return sorted(apps, key=lambda k: k["name"])
+    else:
+        return DISCO_CACHE[discovery_url]