howler-api 3.0.0.dev374__py3-none-any.whl

howler/utils/lucene.py

@@ -0,0 +1,77 @@
+import ipaddress
+import math
+import re
+import sys
+from datetime import datetime
+from typing import Any, Callable, Literal, Optional
+
+from howler.api import Union
+
+
+def try_parse_date(date: str) -> Optional[datetime]:
+    "Try and parse an ISO-formatted string into a date. Returns None if string is invalid."
+    try:
+        # Check if the value is a ISO-formatted date
+        if sys.version_info.major < 11:
+            return datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f%z")
+        else:
+            return datetime.fromisoformat(date)
+    except (ValueError, TypeError):
+        return None
+
+
+def try_parse_number(number: Union[str, int, float]) -> Optional[Union[int, float]]:
+    "Try and parse a number string into an integer or float type, or infinity. Returns None if string is invalid."
+    if isinstance(number, (int, float)):
+        return number
+
+    if number.lower() == "infinity":
+        return math.inf
+
+    try:
+        # Check if the value is an integer
+        return int(number)
+    except ValueError:
+        try:
+            # Check if the value is a float
+            return float(number)
+        except ValueError:
+            return None
+
+
+def try_parse_ip(ip: str) -> Optional[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]]:
+    "Try and parse an ipv4/ipv6 string into a date. Returns None if string is invalid."
+    try:
+        # Check if the value is an IP address
+        return ipaddress.ip_address(ip)
+    except ValueError:
+        return None
+
+
+def coerce(value: Union[list[str], str], fn: Callable[[str], Any]) -> Any:
+    """Coerce a value of list of values using a given function or class.
+
+    Will return an empty list if all results are None.
+    """
+    if isinstance(value, list):
+        result: list = []
+        for _value in value:
+            if fn_result := fn(_value):
+                result.append(fn_result)
+        return result
+    else:
+        return fn(value)
+
+
+def normalize_phrase(value: str, type: Union[Literal["phrase"], Literal["word"]]) -> list[str]:
+    "Normalize a phrase or word for validation"
+    if re.match(r"^[a-zA-Z0-9]+$", value):
+        # probably an ID, no normalization
+        return [value, value.lower()]
+
+    if type == "word":
+        value = re.sub(r"[^a-z0-9.,@_:/;()\-]", "", value.lower())
+    else:
+        value = re.sub(r"[^a-z0-9.,@_:/;()\- ]", "", value, flags=re.IGNORECASE)
+
+    return [value]

howler/utils/path.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import os
+import sys
+from typing import Optional
+
+
+def modulepath(modulename: str) -> str:
+    "Get the path to a given mdule"
+    m = sys.modules[modulename]
+    f = getattr(m, "__file__", None)
+
+    if not f:
+        return os.path.abspath(os.getcwd())
+
+    return os.path.dirname(os.path.abspath(f))
+
+
+def splitpath(path: str, sep: Optional[str] = None) -> list:
+    """Split the path into a list of items"""
+    return list(filter(len, path.split(sep or os.path.sep)))
+
+
+ASCII_NUMBERS = list(range(48, 58))
+ASCII_UPPER_CASE_LETTERS = list(range(65, 91))
+ASCII_LOWER_CASE_LETTERS = list(range(97, 123))
+ASCII_OTHER = [45, 46, 92]  # "-", ".", and "\"

howler/utils/socket_utils.py

@@ -0,0 +1,61 @@
+from howler.common.logging import get_logger
+from howler.datastore.operations import OdmHelper
+from howler.odm.models.hit import Hit
+from howler.services import event_service, hit_service
+
+logger = get_logger(__file__)
+
+hit_helper = OdmHelper(Hit)
+
+
+def check_action(
+    id: str, action: str, broadcast: bool, outstanding_actions: list[tuple[str, str, bool]] = [], **kwargs
+) -> list[tuple[str, str, bool]]:
+    """Emit an event based on the specified action for use by websocket clients
+
+    Args:
+        id (str): The id of the item the action is being run on
+        action (str): The action we are running
+        broadcast (bool): Whether to advertise this action to other users
+        outstanding_actions (list[tuple[str, str, bool]], optional): A list of actions that must be run after the
+        user is disconnected. Defaults to [].
+
+    Returns:
+        list[tuple[str, str, bool]]: The new list of outstanding actions
+    """
+    if broadcast:
+        event_service.emit(
+            "broadcast",
+            {"id": id, "action": action, "username": kwargs["username"]},
+        )
+
+    if action == "typing":
+        outstanding_actions.append((id, "stop_typing", True))
+    elif action == "stop_typing":
+        outstanding_actions = [a for a in outstanding_actions if a[1] != "stop_typing"]
+
+    elif action == "viewing":
+        if hit_service.exists(id):
+            outstanding_actions.append((id, "stop_viewing", False))
+            hit_service.update_hit(
+                id,
+                [
+                    hit_helper.list_add(
+                        "howler.viewers",
+                        kwargs["username"],
+                        silent=True,
+                        if_missing=True,
+                    )
+                ],
+                user=kwargs["username"],
+            )
+    elif action == "stop_viewing":
+        if hit_service.exists(id):
+            hit_service.update_hit(
+                id,
+                [hit_helper.list_remove("howler.viewers", kwargs["username"], silent=True)],
+                user=kwargs["username"],
+            )
+        outstanding_actions = [a for a in outstanding_actions if a[1] != "stop_viewing"]
+
+    return outstanding_actions

howler/utils/str_utils.py

@@ -0,0 +1,256 @@
+import os
+import re
+from copy import copy
+from typing import Any, Optional, Union
+
+import chardet
+
+from howler.common.exceptions import HowlerAttributeError, HowlerTypeError
+
+
+def remove_bidir_unicode_controls(in_str):
+    """Remove UBA characters"""
+    try:
+        no_controls_str = "".join(
+            c
+            for c in in_str
+            if c
+            not in [
+                "\u202e",
+                "\u202b",
+                "\u202d",
+                "\u202a",
+                "\u200e",
+                "\u200f",
+            ]
+        )
+    except Exception:
+        no_controls_str = in_str
+
+    return no_controls_str
+
+
+def wrap_bidir_unicode_string(uni_str):
+    """Wraps str in a LRE (Left-to-Right Embed) unicode control.
+
+    Guarantees that str can be concatenated to other strings without
+    affecting their left-to-right direction
+    """
+    if len(uni_str) == 0 or isinstance(uni_str, bytes):  # Not str, return it unchanged
+        return uni_str
+
+    re_obj = re.search(r"[\u202E\u202B\u202D\u202A\u200E\u200F]", uni_str)
+    if re_obj is None or len(re_obj.group()) == 0:  # No unicode bidir controls found, return string unchanged
+        return uni_str
+
+    # Parse str for unclosed bidir blocks
+    count = 0
+    for letter in uni_str:
+        if letter in ["\u202a", "\u202b", "\u202d", "\u202e"]:  # bidir block open?
+            count += 1
+        elif letter == "\u202c":
+            if count > 0:
+                count -= 1
+
+    # close all bidir blocks
+    if count > 0:
+        uni_str += "\u202c" * count
+
+        # Final wrapper (LTR block) to neutralize any Marks (u+200E and u+200F)
+    uni_str = "\u202a" + uni_str + "\u202c"
+
+    return uni_str
+
+
+# According to wikipedia, RFC 3629 restricted UTF-8 to end at U+10FFFF.
+# This removed the 6, 5 and (irritatingly) half of the 4 byte sequences.
+#
+# The start byte for 2-byte sequences should be a value between 0xc0 and
+# 0xdf but the values 0xc0 and 0xc1 are invalid as they could only be
+# the result of an overlong encoding of basic ASCII characters. There
+# are similar restrictions on the valid values for 3 and 4-byte sequences.
+_valid_utf8 = re.compile(
+    rb"""((?:
+    [\x09\x0a\x20-\x7e]|             # 1-byte (ASCII excluding control chars).
+    [\xc2-\xdf][\x80-\xbf]|          # 2-bytes (excluding overlong sequences).
+    [\xe0][\xa0-\xbf][\x80-\xbf]|    # 3-bytes (excluding overlong sequences).
+
+    [\xe1-\xec][\x80-\xbf]{2}|       # 3-bytes.
+    [\xed][\x80-\x9f][\x80-\xbf]|    # 3-bytes (up to invalid code points).
+    [\xee-\xef][\x80-\xbf]{2}|       # 3-bytes (after invalid code points).
+
+    [\xf0][\x90-\xbf][\x80-\xbf]{2}| # 4-bytes (excluding overlong sequences).
+    [\xf1-\xf3][\x80-\xbf]{3}|       # 4-bytes.
+    [\xf4][\x80-\x8f][\x80-\xbf]{2}  # 4-bytes (up to U+10FFFF).
+    )+)""",
+    re.VERBOSE,
+)
+
+
+def _escape(t, reversible=True):
+    if t[0] % 2:
+        return t[1].replace(b"\\", b"\\\\") if reversible else t[1]
+    else:
+        return b"".join((b"\\x%02x" % x) for x in t[1])
+
+
+def dotdump(s: Union[bytes, str, list[int]]):
+    """Remove any non-ascii characters and replace them with periods
+
+    https://www.cs.cmu.edu/~pattis/15-1XX/common/handouts/ascii.html
+    """
+    if isinstance(s, str):
+        s = s.encode()
+
+    return "".join(["." if x < 32 or x > 126 else chr(x) for x in s])
+
+
+def escape_str(s, reversible=True, force_str=False):
+    """Escape a string"""
+    if isinstance(s, bytes):
+        return escape_str_strict(s, reversible)
+    elif not isinstance(s, str):
+        if force_str:
+            return str(s)
+        return s
+
+    try:
+        return escape_str_strict(
+            s.encode("utf-16", "surrogatepass").decode("utf-16").encode("utf-8"),
+            reversible,
+        )
+    except Exception:
+        return escape_str_strict(s.encode("utf-8", errors="backslashreplace"), reversible)
+
+
+# Returns a string (str) with only valid UTF-8 byte sequences.
+def escape_str_strict(s: bytes, reversible: bool = True) -> str:
+    """Strictly escape a string"""
+    escaped = b"".join([_escape(t, reversible) for t in enumerate(_valid_utf8.split(s))])
+    return escaped.decode("utf-8")
+
+
+def safe_str(s, force_str=False):
+    """Create a safe, escaped string"""
+    return escape_str(s, reversible=False, force_str=force_str)
+
+
+def is_safe_str(s: str) -> bool:
+    """Check if a given string is safe"""
+    return escape_str(copy(s), reversible=False) == s
+
+
+def translate_str(s: Union[str, bytes], min_confidence: float = 0.7) -> dict:
+    """Translate a string from an arbitrary encoding to a python str"""
+    if not isinstance(s, (str, bytes)):
+        raise HowlerTypeError(f"Expected str or bytes got {type(s)}")
+
+    if isinstance(s, str):
+        s = s.encode("utf-8")
+
+    try:
+        r: Any = chardet.detect(s)
+    except Exception:
+        r = {"confidence": 0.0, "encoding": None, "language": None}
+
+    if r["confidence"] > 0 and r["confidence"] >= min_confidence:
+        try:
+            t: Union[str, bytes] = s.decode(r["encoding"] or "utf-8")
+        except Exception:
+            t = s
+    else:
+        t = s
+
+    r["converted"] = safe_str(t)
+    r["encoding"] = r["encoding"] or "unknown"
+    r["language"] = r["language"] or "unknown"
+
+    return r  # type: ignore
+
+
+# This method not really necessary. More to stop people from rolling their own.
+def unescape_str(s):
+    """unescape a string"""
+    return s.decode("string_escape")
+
+
+def truncate(data: Union[bytes, str], length: int = 100) -> str:
+    """This method is a helper used to avoid cluttering output
+
+    :param data: The buffer that will be determined if it needs to be sliced
+    :param length: The limit of characters to the buffer
+    :return str: The potentially truncated buffer
+    """
+    string = safe_str(data)
+    if len(string) > length:
+        return string[:length] + "..."
+    return string
+
+
+def default_string_value(
+    *values: Optional[str], env_name: Optional[str] = None, default: Optional[str] = None
+) -> Optional[str]:
+    """Return a string value based on a list of potential values, an environmnet variable, or a default string"""
+    return next(
+        (val for val in values if val), (os.getenv(env_name, default or "") or default) if env_name else default
+    )
+
+
+def get_parent_key(key: str) -> str:
+    """Get a parent key of a key in the format a.b.c"""
+    return re.sub(r"^(.+)\..+?$", r"\1", key)
+
+
+def sanitize_lucene_query(query: str):
+    """Take in a given string, and escape it to ensure it is safe to search on via lucene"""
+    query = re.sub(r'([\^"~*?:\\/()[\]{}\-!])', r"\\\1", query)
+
+    return query.replace("&&", "\\&&").replace("||", "\\||")
+
+
+class NamedConstants(object):
+    """A class containing a list of named constants, as well as a reverse map for those constants to their name"""
+
+    def __init__(self, name: str, string_value_list: list[tuple[str, Union[str, int]]]):
+        self._name = name
+        self._value_map = dict(string_value_list)
+        self._reverse_map = dict([(s[1], s[0]) for s in string_value_list])
+
+        # we also import the list as attributes so things like
+        # tab completion and introspection still work.
+        for s, v in self._value_map.items():
+            setattr(self, s, v)
+
+    def name_for_value(self, v):
+        """Get the name of a given value"""
+        return self._reverse_map[v]
+
+    def contains_value(self, v):
+        """Chgeck if this instance contains the given value"""
+        return v in self._reverse_map
+
+    def __getitem__(self, s):
+        return self._value_map[s]
+
+    def __getattr__(self, s):
+        # We implement our own getattr mainly to provide the better exception.
+        return self._value_map[s]
+
+
+class StringTable(NamedConstants):
+    """A subclass of NamedConstants that throws an attribute error if the value does not exist in the table"""
+
+    def contains_string(self, s):
+        """Chgeck if this instance contains the given value"""
+        return s in self._reverse_map
+
+    def __getitem__(self, s):
+        if s in self._value_map:
+            return s
+        raise HowlerAttributeError("Invalid value for %s (%s)" % (self._name, s))
+
+    def __getattr__(self, s):
+        # We implement our own getattr mainly to provide the better exception.
+        if s in self._value_map:
+            return s
+        raise HowlerAttributeError("Invalid value for %s (%s)" % (self._name, s))

howler/utils/uid.py

@@ -0,0 +1,47 @@
+import hashlib
+import uuid
+from typing import Any, Optional
+
+import baseconv
+
+from howler.common.exceptions import HowlerValueError
+
+TINY = 8
+SHORT = 16
+MEDIUM = NORMAL = 32
+LONG = 64
+
+
+def get_random_id() -> str:
+    """Get a random id in base 62
+
+    Returns:
+        str: A random base62 id
+    """
+    return baseconv.base62.encode(uuid.uuid4().int)
+
+
+def get_id_from_data(data: Any, prefix: Optional[str] = None, length: int = MEDIUM) -> str:
+    """Get an ID of a particular length, with an optional prefix, based on the provided data.
+
+    Args:
+        data (Any): The data to generate an ID from
+        prefix (str, optional): The prefix to append to the generated ID. Defaults to None.
+        length (int, optional): The length of the random ID. Defaults to MEDIUM.
+
+    Raises:
+        HowlerValueError: An invalid length was provided.
+
+    Returns:
+        str: A unique hash generated from the provided data
+    """
+    possible_len = [TINY, SHORT, MEDIUM, LONG]
+    if length not in possible_len:
+        raise HowlerValueError(f"Invalid hash length of {length}. Possible values are: {str(possible_len)}.")
+    sha256_hash = hashlib.sha256(str(data).encode()).hexdigest()[:length]
+    _hash = baseconv.base62.encode(int(sha256_hash, 16))
+
+    if isinstance(prefix, str):
+        _hash = f"{prefix}_{_hash}"
+
+    return _hash

howler_api-3.0.0.dev374.dist-info/METADATA

@@ -0,0 +1,71 @@
+Metadata-Version: 2.4
+Name: howler-api
+Version: 3.0.0.dev374
+Summary: Howler - API server
+License: MIT
+Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
+Author: Canadian Centre for Cyber Security
+Author-email: howler@cyber.gc.ca
+Maintainer: Matthew Rafuse
+Maintainer-email: matthew.rafuse@cyber.gc.ca
+Requires-Python: >=3.9.17,<4.0.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: apscheduler (==3.10.4)
+Requires-Dist: authlib (>=1.6.0,<2.0.0)
+Requires-Dist: azure-identity (==1.16.1)
+Requires-Dist: azure-storage-blob (==12.14.1)
+Requires-Dist: chardet (==5.1.0)
+Requires-Dist: chevron (==0.14.0)
+Requires-Dist: elastic-apm[flask] (>=6.22.0,<7.0.0)
+Requires-Dist: elasticsearch (==8.6.1)
+Requires-Dist: flasgger (>=0.9.7.1,<0.10.0.0)
+Requires-Dist: flask (==2.2.5)
+Requires-Dist: flask-caching (==2.0.2)
+Requires-Dist: gevent (==23.9.1)
+Requires-Dist: gunicorn (==23.0.0)
+Requires-Dist: luqum (>=1.0.0,<2.0.0)
+Requires-Dist: mergedeep (>=1.3.4,<2.0.0)
+Requires-Dist: packaging (<25.0)
+Requires-Dist: passlib (==1.7.4)
+Requires-Dist: prometheus-client (==0.17.1)
+Requires-Dist: pydantic (>=2.11.4,<3.0.0)
+Requires-Dist: pydantic-settings[yaml] (>=2.9.1,<3.0.0)
+Requires-Dist: pydash (>=8.0.5,<9.0.0)
+Requires-Dist: pyjwt (==2.6.0)
+Requires-Dist: pyroute2-core (==0.6.13)
+Requires-Dist: pysftp (==0.2.9)
+Requires-Dist: pysigma (==0.11.17)
+Requires-Dist: pysigma-backend-elasticsearch (>=1.1.2,<2.0.0)
+Requires-Dist: python-baseconv (==1.2.2)
+Requires-Dist: python-datemath (==3.0.3)
+Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
+Requires-Dist: pyyaml (==6.0.2)
+Requires-Dist: redis (==4.5.4)
+Requires-Dist: requests (==2.32.4)
+Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
+Requires-Dist: validators (>=0.34.0,<0.35.0)
+Requires-Dist: wsproto (==1.2.0)
+Project-URL: Documentation, https://cybercentrecanada.github.io/howler/developer/backend/
+Project-URL: Homepage, https://cybercentrecanada.github.io/howler/
+Project-URL: Repository, https://github.com/CybercentreCanada/howler-api
+Description-Content-Type: text/markdown
+
+# Howler API
+
+## Introduction
+
+Howler is an application that allows analysts to triage hits and alerts. It provides a way for analysts to efficiently review and analyze alerts generated by different analytics and detections.
+
+## Contributing
+
+See [CONTRIBUTING.md](doc/CONTRIBUTING.md).
+