howler-api 3.2.0.dev561__tar.gz → 3.2.0.dev576__tar.gz

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 3.2.0.dev561
+Version: 3.2.0.dev576
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/datastore/types.py

@@ -1,4 +1,10 @@
-from typing import Generic, NotRequired, TypedDict, TypeVar
+import sys
+from typing import Generic, TypedDict, TypeVar
+
+if sys.version_info >= (3, 11):
+    from typing import NotRequired
+else:
+    from typing_extensions import NotRequired
 
 SearchResultType = TypeVar("SearchResultType")
 

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/odm/models/config.py

@@ -28,6 +28,12 @@ class RedisServer(BaseModel):
 
     host: str = Field(description="Hostname of Redis instance")
     port: int = Field(description="Port of Redis instance")
+    password: Optional[str] = Field(description="Password for Redis instance", default=None)
+    tls_enabled: bool = Field(default=False, description="Enable TLS for Redis connection")
+    tls_ca_cert: Optional[str] = Field(
+        description="Path to CA Certificate (PEM) to validate Redis instance certificate when using TLS", default=None
+    )
+    is_cluster: bool = Field(default=False, description="Is this Redis instance a cluster?")
 
 
 class Redis(BaseModel):

howler_api-3.2.0.dev576/howler/remote/datatypes/__init__.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+
+import json
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+
+import redis
+from packaging.version import parse
+
+from howler.common import loader
+from howler.odm.models.config import config
+from howler.utils.uid import get_random_id
+
+# Add a version warning if redis python client is < 2.10.0. Older versions
+# have a connection bug that can manifest with the dispatcher.
+if parse(redis.__version__) <= parse("2.10.0"):
+    import warnings
+
+    warnings.warn(
+        "%s works best with redis > 2.10.0. You're running"
+        " redis %s. You should upgrade." % (__name__, redis.__version__)
+    )
+
+
+log = logging.getLogger(f"{loader.APP_NAME}.queue")
+pool: dict[tuple[str, str, bool], redis.BlockingConnectionPool] = {}
+
+
+def now_as_iso():
+    s = datetime.utcfromtimestamp(time.time()).isoformat()
+    return f"{s}Z"
+
+
+def reply_queue_name(prefix=None, suffix=None):
+    if prefix:
+        components = [prefix]
+    else:
+        components = []
+
+    components.append(get_random_id())
+
+    if suffix:
+        components.append(str(suffix))
+
+    return "-".join(components)
+
+
+def retry_call(func, *args, **kw):
+    maximum = 2
+    exponent = -7
+
+    while True:
+        try:
+            ret_val = func(*args, **kw)
+
+            if exponent != -7:
+                log.info("Reconnected to Redis!")
+
+            return ret_val
+        except (redis.ConnectionError, ConnectionResetError) as ce:
+            log.warning(f"No connection to Redis, reconnecting... [{ce}]")
+            time.sleep(2**exponent)
+            exponent = exponent + 1 if exponent < maximum else exponent
+
+
+def get_client(
+    host: str | redis.Redis | redis.StrictRedis | redis.RedisCluster | None, port: int | None, private: bool = False
+):
+    """
+    Get Redis instance.
+
+    Args:
+        host: Redis host, defaults to nonpersistent host
+        port: Redis port, defaults to nonpersistent port
+        private: If true then use standard connection, otherwise use a Pool
+
+    Returns:
+        Redis instance
+    """
+    # In case a structure is passed a client as host
+    if isinstance(host, (redis.Redis, redis.StrictRedis, redis.RedisCluster)):
+        return host
+
+    if not host or not port:
+        host = host or config.core.redis.nonpersistent.host
+        port = int(port or config.core.redis.nonpersistent.port)
+
+    extra_conn_config: dict[str, str | bool | int] = {}
+    host_config = None
+
+    if host == config.core.redis.nonpersistent.host and port == config.core.redis.nonpersistent.port:
+        host_config = config.core.redis.nonpersistent
+    else:
+        host_config = config.core.redis.persistent
+
+    if host_config.password:
+        extra_conn_config["username"] = "default"
+        extra_conn_config["password"] = host_config.password
+
+    if host_config.tls_enabled:
+        extra_conn_config["ssl"] = True
+        extra_conn_config["ssl_cert_reqs"] = "required"
+
+        if host_config.tls_ca_cert:
+            if not Path(host_config.tls_ca_cert).exists():
+                raise FileNotFoundError(f"Redis TLS CA cert or path '{host_config.tls_ca_cert}' not found.")
+
+            if Path(host_config.tls_ca_cert).is_file():
+                extra_conn_config["ssl_ca_certs"] = host_config.tls_ca_cert
+            else:
+                extra_conn_config["ssl_ca_path"] = host_config.tls_ca_cert
+
+    if host_config.is_cluster is True:
+        return redis.RedisCluster(host=host, port=port, **extra_conn_config)  # type: ignore
+
+    if private:
+        return redis.StrictRedis(host=host, port=port, **extra_conn_config)  # type: ignore
+    else:
+        return redis.StrictRedis(connection_pool=get_pool(host, port, **extra_conn_config))
+
+
+def get_pool(host, port, **kwargs):
+    """
+    Get Redis connection pool
+    Args:
+        host: Redis host
+        port: Redis port
+        **kwargs: Extra parameters to pass to pool connection class
+
+    Returns:
+        Redis BlockingConnectionPool
+    """
+    key = (host, str(port), kwargs.get("ssl", False))
+    connection_pool = pool.get(key, None)
+
+    if not connection_pool:
+        if "ssl" in kwargs and kwargs["ssl"]:
+            # SSLConnection class doesn't accept 'ssl' parameter as it implicitly uses SSL
+            kwargs.pop("ssl")
+            connection_pool = redis.BlockingConnectionPool(
+                host=host, port=port, max_connections=200, connection_class=redis.SSLConnection, **kwargs
+            )
+        else:
+            connection_pool = redis.BlockingConnectionPool(host=host, port=port, max_connections=200, **kwargs)
+        pool[key] = connection_pool
+
+    return connection_pool
+
+
+def decode(data):
+    try:
+        return json.loads(data)
+    except ValueError:
+        log.warning("Invalid data on queue: %s", str(data))
+        return None

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/events.py

@@ -8,7 +8,7 @@ from howler.common.logging import get_logger
 from howler.remote.datatypes import get_client, retry_call
 
 if TYPE_CHECKING:
-    from redis import Redis
+    pass
 
 
 logger = get_logger(__name__)
@@ -21,12 +21,12 @@ class EventSender(Generic[MessageType]):
     def __init__(
         self,
         prefix: str,
-        host=None,
-        port=None,
-        private=None,
+        host: str | None = None,
+        port: int | None = None,
+        private: bool | None = None,
         serializer: Callable[[MessageType], str] = json.dumps,
     ):
-        self.client: Redis[Any] = get_client(host, port, private)
+        self.client = get_client(host, port, bool(private))
         self.prefix = prefix.lower()
         if not self.prefix.endswith("."):
             self.prefix += "."
@@ -40,12 +40,12 @@ class EventSender(Generic[MessageType]):
 class EventWatcher(Generic[MessageType]):
     def __init__(
         self,
-        host=None,
-        port=None,
+        host: str | None = None,
+        port: int | None = None,
         private=None,
         deserializer: Callable[[str], MessageType] = json.loads,
     ):
-        client: Redis[Any] = get_client(host, port, private)
+        client = get_client(host, port, bool(private))
         self.pubsub = retry_call(client.pubsub)
         self.worker: Optional[threading.Thread] = None
         self.deserializer = deserializer

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/hash.py

@@ -2,7 +2,9 @@ from __future__ import annotations
 
 import json
 import time
-from typing import TYPE_CHECKING, Generic, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar
+
+from redis import RedisCluster
 
 from howler.common.exceptions import HowlerValueError
 from howler.remote.datatypes import get_client, retry_call
@@ -71,10 +73,10 @@ class Hash(Generic[T]):
     def __init__(
         self,
         name: str,
-        host: Optional[Union[str, Redis]] = None,
+        host: str | Redis[Any] | RedisCluster[Any] | None = None,
         port: Optional[int] = None,
     ):
-        self.c = get_client(host, port, False)
+        self.c: Any = get_client(host, port, False)
         self.name = name
         self._pop = self.c.register_script(h_pop_script)
         self._limited_add = self.c.register_script(_limited_add)

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/lock.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from howler.remote.datatypes import get_client, retry_call
 from howler.utils.uid import get_random_id
 
@@ -27,7 +29,7 @@ end
 class Lock(object):
     def __init__(self, name, timeout, host=None, port=None):
         self.uuid = get_random_id()
-        self.c = get_client(host, port, False)
+        self.c: Any = get_client(host, port, False)
         self.lock_release = "-".join(("lock", str(timeout), name, "released"))
         self.lock_holder = "-".join(("lock", str(timeout), name, "holder"))
         self.timeout = timeout

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/queues/multi.py

@@ -1,11 +1,12 @@
 import json
+from typing import Any
 
 from howler.remote.datatypes import get_client, retry_call
 
 
 class MultiQueue(object):
     def __init__(self, host=None, port=None, private=False):
-        self.c = get_client(host, port, private)
+        self.c: Any = get_client(host, port, private)
 
     def delete(self, name):
         retry_call(self.c.delete, name)

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/queues/named.py

@@ -1,6 +1,6 @@
 import json
 import time
-from typing import Generic, Optional, TypeVar
+from typing import Any, Generic, Optional, TypeVar
 
 from howler.common.exceptions import HowlerTypeError
 from howler.remote.datatypes import get_client, retry_call
@@ -10,7 +10,7 @@ T = TypeVar("T")
 
 class NamedQueue(Generic[T]):
     def __init__(self, name: str, host=None, port=None, private: bool = False, ttl: int = 0):
-        self.c = get_client(host, port, private)
+        self.c: Any = get_client(host, port, private)
         self.name: str = name
         self.ttl: int = ttl
         self.last_expire_time: float = 0

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/queues/priority.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 import time
-from typing import Generic, Optional, TypeVar, Union
+from typing import Any, Generic, Optional, TypeVar, Union
 
 from howler.common.exceptions import HowlerTypeError
 from howler.remote.datatypes import decode, get_client, retry_call
@@ -36,7 +36,7 @@ T = TypeVar("T")
 
 class PriorityQueue(Generic[T]):
     def __init__(self, name, host=None, port=None, private=False):
-        self.c = get_client(host, port, private)
+        self.c: Any = get_client(host, port, private)
         self._deque_range = self.c.register_script(pq_dequeue_range_script)
         self.name = name
 

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/set.py

@@ -1,5 +1,6 @@
 import json
 import time
+from typing import Any
 
 from howler.remote.datatypes import get_client, retry_call
 
@@ -26,7 +27,7 @@ return false
 
 class Set(object):
     def __init__(self, name, host=None, port=None):
-        self.c = get_client(host, port, False)
+        self.c: Any = get_client(host, port, False)
         self.name = name
         self._drop_card = self.c.register_script(_drop_card_script)
         self._limited_add = self.c.register_script(_limited_add)

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/howler/remote/datatypes/user_quota_tracker.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 import redis
 
 from howler.remote.datatypes import get_client, retry_call
@@ -22,7 +24,7 @@ end
 
 class UserQuotaTracker(object):
     def __init__(self, prefix, timeout=120, redis=None, host=None, port=None, private=False):
-        self.c = redis or get_client(host, port, private)
+        self.c: Any = redis or get_client(host, port, private)
         self.bs = self.c.register_script(begin_script)
         self.prefix = prefix
         self.timeout = timeout

{howler_api-3.2.0.dev561 → howler_api-3.2.0.dev576}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "3.2.0.dev561"
+version = "3.2.0.dev576"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",
@@ -240,7 +240,7 @@ mypy = "^1.6.1"
 pytest = "^8.1.1"
 retrying = "^1.3.4"
 pyftpdlib = "^1.5.9"
-pyopenssl = "^24.1.0"
+pyopenssl = ">=24.1,<27.0"
 diff-cover = "^9.2.4"
 pytest-cov = "^4.1.0"
 mock = "^5.1.0"

howler_api-3.2.0.dev561/howler/remote/datatypes/__init__.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-
-import json
-import logging
-import time
-from datetime import datetime
-
-import redis
-from packaging.version import parse
-
-from howler.common import loader
-from howler.odm.models.config import config
-from howler.utils.uid import get_random_id
-
-# Add a version warning if redis python client is < 2.10.0. Older versions
-# have a connection bug that can manifest with the dispatcher.
-if parse(redis.__version__) <= parse("2.10.0"):
-    import warnings
-
-    warnings.warn(
-        "%s works best with redis > 2.10.0. You're running"
-        " redis %s. You should upgrade." % (__name__, redis.__version__)
-    )
-
-
-log = logging.getLogger(f"{loader.APP_NAME}.queue")
-pool: dict[tuple[str, str], redis.BlockingConnectionPool] = {}
-
-
-def now_as_iso():
-    s = datetime.utcfromtimestamp(time.time()).isoformat()
-    return f"{s}Z"
-
-
-def reply_queue_name(prefix=None, suffix=None):
-    if prefix:
-        components = [prefix]
-    else:
-        components = []
-
-    components.append(get_random_id())
-
-    if suffix:
-        components.append(str(suffix))
-
-    return "-".join(components)
-
-
-def retry_call(func, *args, **kw):
-    maximum = 2
-    exponent = -7
-
-    while True:
-        try:
-            ret_val = func(*args, **kw)
-
-            if exponent != -7:
-                log.info("Reconnected to Redis!")
-
-            return ret_val
-        except (redis.ConnectionError, ConnectionResetError) as ce:
-            log.warning(f"No connection to Redis, reconnecting... [{ce}]")
-            time.sleep(2**exponent)
-            exponent = exponent + 1 if exponent < maximum else exponent
-
-
-def get_client(host, port, private):
-    # In case a structure is passed a client as host
-    if isinstance(host, (redis.Redis, redis.StrictRedis)):
-        return host
-
-    if not host or not port:
-        host = host or config.core.redis.nonpersistent.host
-        port = int(port or config.core.redis.nonpersistent.port)
-
-    if private:
-        return redis.StrictRedis(host=host, port=port)
-    else:
-        return redis.StrictRedis(connection_pool=get_pool(host, port))
-
-
-def get_pool(host, port):
-    key = (host, port)
-
-    connection_pool = pool.get(key, None)
-    if not connection_pool:
-        connection_pool = redis.BlockingConnectionPool(host=host, port=port, max_connections=200)
-        pool[key] = connection_pool
-
-    return connection_pool
-
-
-def decode(data):
-    try:
-        return json.loads(data)
-    except ValueError:
-        log.warning("Invalid data on queue: %s", str(data))
-        return None