howler-api 4.0.0.dev1141__tar.gz → 4.0.0.dev1147__tar.gz

{howler_api-4.0.0.dev1141 → howler_api-4.0.0.dev1147}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev1141
+Version: 4.0.0.dev1147
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: apscheduler (==3.11.2)
+Requires-Dist: apscheduler (==3.11.3)
 Requires-Dist: authlib (>=1.6.0,<2.0.0)
 Requires-Dist: azure-monitor-opentelemetry (>=1.8.7,<2.0.0)
 Requires-Dist: bcrypt (==4.3.0)
@@ -50,7 +50,7 @@ Requires-Dist: python-datemath (==3.0.3)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
 Requires-Dist: pytz (>=2025.2,<2026.0)
 Requires-Dist: pyyaml (==6.0.3)
-Requires-Dist: redis (==4.6.0)
+Requires-Dist: redis (==8.0.1)
 Requires-Dist: requests (==2.34.2)
 Requires-Dist: tzdata (>=2026.1,<2027.0)
 Requires-Dist: validators (>=0.34,<0.36)

howler_api-4.0.0.dev1147/howler/healthz.py

@@ -0,0 +1,91 @@
+from flask import Blueprint, abort, make_response
+from redis import RedisError
+
+from howler import config as howler_config
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+
+API_PREFIX = "/api/healthz"
+healthz = Blueprint("healthz", __name__, url_prefix=API_PREFIX)
+logger = get_logger(__file__)
+
+
+def _is_datastore_healthy() -> bool:
+    """Check whether the datastore is reachable."""
+    try:
+        if datastore().ds.ping():
+            return True
+    except Exception as error:  # noqa: BLE001
+        logger.warning("Health check failed: datastore ping raised %s", error)
+        return False
+
+    logger.warning("Health check failed: datastore ping returned false")
+    return False
+
+
+def _is_redis_healthy() -> bool:
+    """Check whether configured Redis clients are reachable."""
+    redis_config = howler_config.config.core.redis
+
+    if redis_config.nonpersistent.host:
+        try:
+            if not howler_config.redis.ping():
+                logger.warning("Health check failed: nonpersistent Redis ping returned false")
+                return False
+        except (ConnectionResetError, OSError, RedisError) as error:
+            logger.warning("Health check failed: nonpersistent Redis ping raised %s", error)
+            return False
+
+    if redis_config.persistent.host:
+        try:
+            if not howler_config.redis_persistent.ping():
+                logger.warning("Health check failed: persistent Redis ping returned false")
+                return False
+        except (ConnectionResetError, OSError, RedisError) as error:
+            logger.warning("Health check failed: persistent Redis ping raised %s", error)
+            return False
+
+    return True
+
+
+@healthz.route("/live")
+def liveness(**_):
+    """Check if the API is live
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    OK or FAIL
+    """
+    return make_response("OK")
+
+
+@healthz.route("/ready")
+def readyness(**_):
+    """Check if the API is Ready
+
+    Checks that the datastore and configured Redis connections are alive.
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    OK or FAIL
+    """
+    if _is_datastore_healthy() and _is_redis_healthy():
+        return make_response("OK")
+    else:
+        abort(503)
+
+
+@healthz.errorhandler(503)
+def error(_):
+    "Handle errors exposed in healthz routes"
+    return "FAIL", 503

{howler_api-4.0.0.dev1141 → howler_api-4.0.0.dev1147}/howler/odm/models/config.py

@@ -33,6 +33,7 @@ class RedisServer(BaseModel):
     tls_ca_cert: Optional[str] = Field(
         description="Path to CA Certificate (PEM) to validate Redis instance certificate when using TLS", default=None
     )
+    tls_disable_check_hostname: bool = Field(default=False, description="Disable TLS hostname check")
     is_cluster: bool = Field(default=False, description="Is this Redis instance a cluster?")
 
 

{howler_api-4.0.0.dev1141 → howler_api-4.0.0.dev1147}/howler/odm/models/view.py

@@ -1,14 +1,29 @@
 # mypy: ignore-errors
-from typing import Literal, Union
+from typing import Literal, Optional, Union
 
 from howler import odm
 
 
+@odm.model(index=True, store=True, description="The field and width of a column to display in a grid view.")
+class GridColumn(odm.Model):
+    field: str = odm.Keyword(description="The field key for this column.")
+    width: Optional[int] = odm.Optional(odm.Integer(description="The width of this column in pixels."))
+
+
 @odm.model(index=True, store=True, description="Additional View Settings")
 class Settings(odm.Model):
     advance_on_triage: bool = odm.Boolean(
         description="Should the user advance to the next alert when triage is complete?", default=False
     )
+    display: Optional[Union[Literal["list"], Literal["grid"]]] = odm.Optional(
+        odm.Enum(
+            values=["list", "grid"],
+            description="The layout to use when opening this view",
+        )
+    )
+    columns: Optional[list[GridColumn]] = odm.Optional(
+        odm.List(odm.Compound(GridColumn, description="The columns to display in this view."))
+    )
 
 
 DEFAULT_INDEXES = ["hit"]

howler_api-4.0.0.dev1147/howler/remote/datatypes/__init__.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+
+import json
+import logging
+import os
+import socket
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import redis
+from packaging.version import parse
+from redis.backoff import ExponentialBackoff
+from redis.retry import Retry
+
+from howler.odm.models.config import config
+from howler.utils.uid import get_random_id
+
+# Add a version warning if redis python client is < 2.10.0. Older versions
+# have a connection bug that can manifest with the dispatcher.
+if parse(redis.__version__) <= parse("2.10.0"):
+    import warnings
+
+    warnings.warn(
+        "%s works best with redis > 2.10.0. You're running"
+        " redis %s. You should upgrade." % (__name__, redis.__version__)
+    )
+
+APP_NAME = os.environ.get("APP_NAME", "howler")
+
+log = logging.getLogger(f"{APP_NAME}.queue")
+pool: dict[tuple[str, str, bool], redis.BlockingConnectionPool] = {}
+
+# TCP keepalive tuning so a connection to a primary that has gone away (e.g.
+# during an Azure Managed Redis failover) is detected in ~90s instead of the
+# Linux default keepalive idle of ~2 hours. This is the mechanism that lets
+# *blocking* reads (BLPOP/BZPOPMIN with timeout=0) notice a dead peer, since we
+# deliberately do NOT set socket_timeout (see RESILIENCE_CONFIG below). The
+# option constants are platform specific, so each is added only if available.
+_KEEPALIVE_OPTIONS: dict[int, int] = {
+    getattr(socket, name): value
+    for name, value in (("TCP_KEEPIDLE", 60), ("TCP_KEEPINTVL", 10), ("TCP_KEEPCNT", 3))
+    if hasattr(socket, name)
+}
+
+# Connection resilience defaults. These ensure that during a managed Redis
+# failover (e.g. Azure Managed Redis) the client quickly detects dead/stale
+# connections and reconnects to the new primary instead of hanging on the OS
+# level TCP timeout (which can be several minutes).
+#   - socket_connect_timeout: cap how long establishing a new connection can
+#     block so a dead endpoint surfaces as a (retryable) error in seconds.
+#   - socket_keepalive / socket_keepalive_options: enable and tune TCP
+#     keepalives so half-open connections (including those blocked in a long
+#     BLPOP/BZPOPMIN) are detected and recycled.
+#   - health_check_interval: proactively PING idle pooled connections before
+#     use so stale connections to the old primary are recycled transparently.
+#
+# NOTE: socket_timeout is explicitly set to None. It is a per-read socket
+# timeout that redis-py also applies to blocking commands, so any positive value
+# makes BLPOP/BZPOPMIN/BRPOP with an infinite (timeout=0) wait raise spurious
+# TimeoutErrors (see redis-py #2807). This MUST be set explicitly: redis-py 8.0
+# changed the default socket_timeout from None to 5 seconds, so simply omitting
+# it would silently break the infinite blocking pops the queue consumers in this
+# package rely on. With socket_timeout disabled, keepalives + health checks
+# handle failover detection instead.
+RESILIENCE_CONFIG: dict[str, Any] = {
+    "socket_connect_timeout": 5,
+    "socket_timeout": None,
+    "socket_keepalive": True,
+    "health_check_interval": 15,
+}
+if _KEEPALIVE_OPTIONS:
+    RESILIENCE_CONFIG["socket_keepalive_options"] = _KEEPALIVE_OPTIONS
+
+# Maximum wall-clock time (in seconds) that retry_call will keep retrying after
+# the first observed Redis connection failure before giving up and re-raising the
+# last error. The retry window starts after the first error because blocking
+# Redis reads rely on TCP keepalive to notice a dead peer, and that detection can
+# take longer than this retry budget.
+RETRY_DEADLINE = 60
+
+
+def now_as_iso():
+    s = datetime.now(timezone.utc).replace(tzinfo=None).isoformat()
+    return f"{s}Z"
+
+
+def reply_queue_name(prefix=None, suffix=None):
+    if prefix:
+        components = [prefix]
+    else:
+        components = []
+
+    components.append(get_random_id())
+
+    if suffix:
+        components.append(str(suffix))
+
+    return "-".join(components)
+
+
+def retry_call(func, *args, **kw):
+    maximum = 2
+    exponent = -7
+    deadline = None
+
+    while True:
+        try:
+            ret_val = func(*args, **kw)
+
+            if exponent != -7:
+                log.info("Reconnected to Redis!")
+
+            return ret_val
+        except (redis.ConnectionError, redis.TimeoutError, OSError) as ce:
+            # redis.ConnectionError covers BusyLoadingError (node loading after a
+            # failover) and most connect-time socket errors. redis.TimeoutError
+            # covers Redis read/connect timeouts. OSError covers the builtin socket
+            # errors raised while a managed Redis instance is failing over,
+            # including its ConnectionResetError and TimeoutError subclasses.
+            #
+            # Use a monotonic clock for the retry budget so NTP/system clock
+            # adjustments can't make retries give up early or run far too long.
+            if deadline is None:
+                deadline = time.monotonic() + RETRY_DEADLINE
+            elif time.monotonic() >= deadline:
+                # Give up after RETRY_DEADLINE so a sustained outage fails the
+                # request cleanly instead of blocking the worker indefinitely.
+                log.exception("No connection to Redis after %ss, giving up.", RETRY_DEADLINE)
+                raise
+
+            log.warning("No connection to Redis, reconnecting... [%s]", ce)
+            time.sleep(2**exponent)
+            exponent = exponent + 1 if exponent < maximum else exponent
+
+
+def get_client(
+    host: str | redis.Redis | redis.StrictRedis | redis.RedisCluster | None,
+    port: int | None,
+    private: bool = False,
+):
+    """
+    Get Redis instance.
+
+    Args:
+        host: Redis host, defaults to nonpersistent host
+        port: Redis port, defaults to nonpersistent port
+        private: If true then use standard connection, otherwise use a Pool
+
+    Returns:
+        Redis instance
+    """
+    # In case a structure is passed a client as host
+    if isinstance(host, (redis.Redis, redis.StrictRedis, redis.RedisCluster)):
+        return host
+
+    if not host or not port:
+        host = host or config.core.redis.nonpersistent.host
+        port = int(port or config.core.redis.nonpersistent.port)
+
+    extra_conn_config: dict[str, Any] = {}
+    host_config = None
+
+    # Apply socket connect timeout / keepalive / health-check defaults so the
+    # client can detect and recover from managed Redis failovers instead of
+    # hanging.
+    extra_conn_config.update(RESILIENCE_CONFIG)
+
+    # Pin RESP2 on the wire. redis-py 8.0 switched the default to RESP3; pinning
+    # protocol=2 preserves the exact wire behaviour this codebase was written
+    # against so the dependency upgrade is behaviour-neutral. Can be revisited to
+    # adopt RESP3 once its response shapes have been validated end to end.
+    extra_conn_config["protocol"] = 2
+
+    # Configure connection-level retries using the modern Retry API (the
+    # replacement for the deprecated retry_on_timeout flag). A fresh Retry
+    # instance is built here per get_client call. Note that for pooled clients
+    # (private=False) the underlying connection pool is cached by host/port/ssl
+    # in get_pool, so this Retry is only applied when the pool is first created
+    # and is then effectively shared (per-pool) by every later client on that
+    # pool. The Retry config itself is immutable, so this sharing is benign.
+    extra_conn_config["retry"] = Retry(ExponentialBackoff(cap=10, base=0.5), retries=3)
+    extra_conn_config["retry_on_error"] = [redis.ConnectionError, redis.TimeoutError]
+
+    if host == config.core.redis.nonpersistent.host and port == config.core.redis.nonpersistent.port:
+        host_config = config.core.redis.nonpersistent
+    else:
+        host_config = config.core.redis.persistent
+
+    if host_config.password:
+        extra_conn_config["username"] = "default"
+        extra_conn_config["password"] = host_config.password
+
+    if host_config.tls_enabled:
+        extra_conn_config["ssl"] = True
+        extra_conn_config["ssl_cert_reqs"] = "required"
+
+        if host_config.tls_disable_check_hostname:
+            extra_conn_config["ssl_check_hostname"] = False
+
+        if host_config.tls_ca_cert:
+            if not Path(host_config.tls_ca_cert).exists():
+                raise FileNotFoundError(f"Redis TLS CA cert or path '{host_config.tls_ca_cert}' not found.")
+
+            if Path(host_config.tls_ca_cert).is_file():
+                extra_conn_config["ssl_ca_certs"] = host_config.tls_ca_cert
+            else:
+                extra_conn_config["ssl_ca_path"] = host_config.tls_ca_cert
+
+    if host_config.is_cluster is True:
+        return redis.RedisCluster(host=host, port=port, **extra_conn_config)  # type: ignore
+
+    if private:
+        return redis.StrictRedis(host=host, port=port, **extra_conn_config)  # type: ignore
+    else:
+        return redis.StrictRedis(connection_pool=get_pool(host, port, **extra_conn_config))
+
+
+def get_pool(host, port, **kwargs):
+    """
+    Get Redis connection pool
+    Args:
+        host: Redis host
+        port: Redis port
+        **kwargs: Extra parameters to pass to pool connection class
+
+    Returns:
+        Redis BlockingConnectionPool
+    """
+    key = (host, str(port), kwargs.get("ssl", False))
+    connection_pool = pool.get(key, None)
+
+    if not connection_pool:
+        if "ssl" in kwargs and kwargs["ssl"]:
+            # SSLConnection class doesn't accept 'ssl' parameter as it implicitly uses SSL
+            kwargs.pop("ssl")
+            connection_pool = redis.BlockingConnectionPool(
+                host=host,
+                port=port,
+                max_connections=200,
+                connection_class=redis.SSLConnection,
+                **kwargs,
+            )
+        else:
+            connection_pool = redis.BlockingConnectionPool(host=host, port=port, max_connections=200, **kwargs)
+        pool[key] = connection_pool
+
+    return connection_pool
+
+
+def decode(data):
+    try:
+        return json.loads(data)
+    except ValueError:
+        log.warning("Invalid data on queue: %s", str(data))
+        return None

{howler_api-4.0.0.dev1141 → howler_api-4.0.0.dev1147}/howler/remote/datatypes/counters.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Any, Optional
 
 from redis.exceptions import ConnectionError
 
@@ -8,7 +8,7 @@ from howler.remote.datatypes.hash import Hash
 
 class Counters(object):
     def __init__(self, prefix="counter", host=None, port=None, track_counters=False):
-        self.c = get_client(host, port, False)
+        self.c: Any = get_client(host, port, False)
         self.prefix = prefix
         if track_counters:
             self.tracker: Optional[Hash] = Hash("c-tracker-%s" % prefix, host=host, port=port)

{howler_api-4.0.0.dev1141 → howler_api-4.0.0.dev1147}/howler/remote/datatypes/hash.py

@@ -73,7 +73,7 @@ class Hash(Generic[T]):
     def __init__(
         self,
         name: str,
-        host: str | Redis[Any] | RedisCluster[Any] | None = None,
+        host: str | Redis | RedisCluster | None = None,
         port: Optional[int] = None,
     ):
         self.c: Any = get_client(host, port, False)

{howler_api-4.0.0.dev1141 → howler_api-4.0.0.dev1147}/howler/remote/datatypes/user_quota_tracker.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-import redis
+from redis.exceptions import ResponseError
 
 from howler.remote.datatypes import get_client, retry_call
 
@@ -35,7 +35,7 @@ class UserQuotaTracker(object):
     def begin(self, user, max_quota):
         try:
             return retry_call(self.bs, args=[self._queue_name(user), max_quota, self.timeout]) == 1
-        except redis.exceptions.ResponseError as er:
+        except ResponseError as er:
             # TODO: This is a failsafe for upgrade purposes could be removed in a future version
             if "WRONGTYPE" in str(er):
                 retry_call(self.c.delete, self._queue_name(user))
@@ -47,7 +47,7 @@ class UserQuotaTracker(object):
         """When only one item is requested, blocking is is possible."""
         try:
             retry_call(self.c.zpopmin, self._queue_name(user))
-        except redis.exceptions.ResponseError as er:
+        except ResponseError as er:
             # TODO: This is a failsafe for upgrade purposes could be removed in a future version
             if "WRONGTYPE" in str(er):
                 retry_call(self.c.delete, self._queue_name(user))

{howler_api-4.0.0.dev1141 → howler_api-4.0.0.dev1147}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev1141"
+version = "4.0.0.dev1147"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",
@@ -197,7 +197,7 @@ repository = "https://github.com/CybercentreCanada/howler-api"
 [tool.poetry.dependencies]
 python = "^3.10"
 gunicorn = "23.0.0"
-apscheduler = "3.11.2"
+apscheduler = "3.11.3"
 authlib = "^1.6.0"
 chardet = "5.2.0"
 elasticsearch = "8.19.3"
@@ -211,7 +211,7 @@ pyjwt = "2.13.0"
 python-baseconv = "1.2.2"
 python-datemath = "3.0.3"
 pyyaml = "6.0.3"
-redis = "4.6.0"
+redis = "8.0.1"
 requests = "2.34.2"
 wsproto = "1.3.2"
 chevron = "0.14.0"
@@ -240,7 +240,7 @@ pyright = { extras = ["nodejs"], version = "^1.1.408" }
 mypy = "^1.6.1"
 
 [tool.poetry.group.test.dependencies]
-pytest = "9.0.3"
+pytest = "9.1.1"
 retrying = "^1.3.4"
 pyftpdlib = "^1.5.9"
 pyopenssl = ">=24.1,<27.0"
@@ -255,7 +255,6 @@ typing-extensions = "^4.12.2"
 types-PyYAML = "6.0.12.20260518"
 types-paramiko = "3.5.0.20250801"
 types-pyOpenSSL = "23.3.0.20240106"
-types-redis = "4.6.0.20241004"
 types-six = "1.17.0.20260518"
 types-requests = "<2.33.0.20260519"
 websocket-client = "1.9.0"

howler_api-4.0.0.dev1141/howler/healthz.py

@@ -1,47 +0,0 @@
-from flask import Blueprint, abort, make_response
-
-from howler.common.loader import datastore
-
-API_PREFIX = "/api/healthz"
-healthz = Blueprint("healthz", __name__, url_prefix=API_PREFIX)
-
-
-@healthz.route("/live")
-def liveness(**_):
-    """Check if the API is live
-
-    Variables:
-    None
-
-    Arguments:
-    None
-
-    Result Example:
-    OK or FAIL
-    """
-    return make_response("OK")
-
-
-@healthz.route("/ready")
-def readyness(**_):
-    """Check if the API is Ready
-
-    Variables:
-    None
-
-    Arguments:
-    None
-
-    Result Example:
-    OK or FAIL
-    """
-    if datastore().ds.ping():
-        return make_response("OK")
-    else:
-        abort(503)
-
-
-@healthz.errorhandler(503)
-def error(_):
-    "Handle errors exposed in healthz routes"
-    return "FAIL", 503