howler-api 3.0.0.dev374__py3-none-any.whl

howler/plugins/config.py

@@ -0,0 +1,123 @@
+import logging
+from typing import Any
+
+from pydantic import BaseModel, ImportString, model_validator
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, YamlConfigSettingsSource
+
+from howler.common.logging import HWL_DATE_FORMAT, HWL_LOG_FORMAT
+
+logger = logging.getLogger("howler.odm.models.config")
+logger.setLevel(logging.INFO)
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+console.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+logger.addHandler(console)
+
+
+class ODMModules(BaseModel):
+    "A set of fields for adding additional fields to Howler's ODM."
+
+    modify_odm: dict[str, ImportString] = {}
+    generation: dict[str, ImportString] = {}
+
+
+class Modules(BaseModel):
+    "A list of components exposed for use in Howler by this plugin."
+
+    routes: list[ImportString] = []
+    operations: list[ImportString] = []
+    token_functions: dict[str, ImportString] = {}
+
+    odm: ODMModules = ODMModules()
+
+
+class BasePluginConfig(BaseSettings):
+    "Configuration File for Plugin"
+
+    name: str
+    features: dict[str, bool] = {}
+
+    modules: Modules = Modules()
+
+    @model_validator(mode="before")
+    @classmethod
+    def initialize_plugin_configuration(cls, data: Any) -> Any:  # noqa: C901
+        "Convert a raw yaml config into an object ready for validation by pydantic"
+        if not isinstance(data, dict):
+            return data
+
+        # Default mutation requires plugin name
+        if "name" not in data:
+            logger.warning("Name is missing from configuration")
+            return data
+
+        plugin_name = data["name"]
+        logger.debug("Beginning configuration parsing for plugin %s", plugin_name)
+
+        if "modules" not in data:
+            return data
+
+        if "routes" in data["modules"] and isinstance(data["modules"]["routes"], list):
+            new_routes: list[str] = []
+            for route in data["modules"]["routes"]:
+                new_routes.append(f"{plugin_name}.routes.{route}" if "." not in route else route)
+
+            data["modules"]["routes"] = new_routes
+
+        if "operations" in data["modules"] and isinstance(data["modules"]["operations"], list):
+            new_operations: list[str] = []
+            for operation in data["modules"]["operations"]:
+                new_operations.append(f"{plugin_name}.actions.{operation}" if "." not in operation else operation)
+
+            data["modules"]["operations"] = new_operations
+
+        if "token_functions" in data["modules"] and isinstance(data["modules"]["token_functions"], dict):
+            new_token_functions_dict: dict[str, str] = {}
+
+            for application, value in data["modules"]["token_functions"].items():
+                if value is True:
+                    new_token_functions_dict[application] = f"{plugin_name}.token.{application}:get_token"
+                elif value is False:
+                    continue
+                else:
+                    new_token_functions_dict[application] = value
+
+            data["modules"]["token_functions"] = new_token_functions_dict
+
+        if "odm" not in data["modules"] or not isinstance(data["modules"]["odm"], dict):
+            return data
+
+        if "modify_odm" in data["modules"]["odm"] and isinstance(data["modules"]["odm"]["modify_odm"], dict):
+            new_modify_odm_dict: dict[str, str] = {}
+            for odm_name, value in data["modules"]["odm"]["modify_odm"].items():
+                if value is True:
+                    new_modify_odm_dict[odm_name] = f"{plugin_name}.odm.{odm_name}:modify_odm"
+                elif value is False:
+                    continue
+                else:
+                    new_modify_odm_dict[odm_name] = value
+
+            data["modules"]["odm"]["modify_odm"] = new_modify_odm_dict
+
+        if "generation" in data["modules"]["odm"] and isinstance(data["modules"]["odm"]["generation"], dict):
+            new_generation_dict: dict[str, str] = {}
+            for odm_name, value in data["modules"]["odm"]["generation"].items():
+                if value is True:
+                    new_generation_dict[odm_name] = f"{plugin_name}.odm.{odm_name}:generate"
+                elif value is False:
+                    continue
+                else:
+                    new_generation_dict[odm_name] = value
+
+            data["modules"]["odm"]["generation"] = new_generation_dict
+
+        return data
+
+    @classmethod
+    def settings_customise_sources(
+        cls,  # noqa: ANN102
+        *args,  # noqa: ANN002
+        **kwargs,  # noqa: ANN002, ANN102
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        "Adds a YamlConfigSettingsSource object at the end of the settings_customize_sources response."
+        return (*super().settings_customise_sources(*args, **kwargs), YamlConfigSettingsSource(cls))

howler/remote/datatypes/README.md

@@ -0,0 +1,355 @@
+# Remote Datatypes
+
+Now that most project are distributed using containers there is sometimes a need for having data structures available live to different process running on different hosts. This is where the remote datatypes com into play. These are essentially data structures stored in Redis that are available to all processes in your cluster.
+
+We have a various range of supported data types to account to various scenarios:
+
+## Counters
+
+Counters are very useful to gather metrics throughout your system.
+
+They support simple integer operation:
+
+- `inc(name, x)`: Increment name counter by X
+- `dec(name, x)`: Decrement name counter by X
+- `set(name, x)`: Set name counter to X
+
+They also have other functions to inspect the values:
+
+- `get_queues()`: List the name of all the counters
+- `get_queues_sizes()`: List the name of all the counters with their current sizes
+- `reset_queues()`: Reset all counters to 0
+- `delete()`: Delete any traces to the counters
+
+Example:
+
+```python
+from howler.remote.datatypes.counters import Counters
+with Counters('test-counter') as ct:
+    # Increment counter by 4
+    ct.inc('value_1', 4)
+
+    # Get Counter values
+    ct.get_queues_sizes()
+```
+
+Output:
+
+```json
+{
+  "test-counter-value_1": 4
+}
+```
+
+## Event Senders/Watchers
+
+Event senders and watchers are use to trigger code execution on multiple container when another container performs an action. If multiple containers register to watch the same event, they will all receive the same message when the event fires.
+
+Methods:
+
+- EventSender:
+  - `send(event_name, event_data)`: Sends the event_data as an event of type event_name
+- EventWatcher:
+  - `register(event_name, callback)`: Register a callback function for all events of type event name (wildcard * can be use to register a single callback to multiple events)
+  - `start()`: Start listening for events
+  - `stop()`: Stop listening for events
+
+Example, A container register a watcher for a specific event:
+
+```python
+import time
+from howler.remote.datatypes.events import EventWatcher
+
+def callback_func(data: dict[str, Any]):
+    print(data)
+
+watcher = EventWatcher()
+try:
+    # register for the test event in the event group
+    watcher.register('event.test', callback_func)
+    watcher.start()
+    while True:
+        time.sleep(1)
+finally:
+    watcher.stop()
+
+```
+
+Another container creates an event to wake up the first container:
+
+```python
+import time
+from howler.remote.datatypes.events import EventSender
+
+# create a sender for the event group
+sender = EventSender('event.', redis_connection)
+
+# send a test even
+sender.send('test', {'payload': 100})
+# After this call, the first container will have woken up and executed its callback function
+# that would print: {'payload': 100}
+```
+
+## Hash Table
+
+Hash tables are used to keep dictionary like structures available to all containers in your cluster.
+
+They support the following methods:
+
+- `add(key, value)`: Add the specified value to the key in the hash table, if the key already exists it is not added
+- `increment(key, count)`: Increment the value at the key by the count
+- `limited_add(key, value, limit)`: Add the specified value to the key in the hash table, if size of the set is smaller then the limit
+- `exists(key)`: Check if a get exists
+- `get(key)`: Get the value at the specfied key
+- `keys()`: Return the list of keys in the hash table
+- `length()`: Return the number of keys in the hash table
+- `items()`: Get the whole hash table keys and values as a dictionary
+- `conditional_remove(key, value)`: Remove the key from the hash table if it value is the same as specified
+- `pop(key)`: Get the value of the key and remove it from the hash table
+- `set(key, value)`: Set the specified key to a specific value
+- `multi_set(dict)`: Set multiple keys at once
+- `delete()`: Completely delete the hash structure
+
+Example:
+
+```python
+from howler.remote.datatypes.hash import Hash
+with Hash('test-hashmap') as h:
+    h.add("key", "value")
+    print(h.exists("key"))
+    print(h.get("key"))
+    print(h.items())
+```
+
+Output:
+
+```python
+True
+"value"
+{"key": "value"}
+```
+
+## Global Locks
+
+If you need to make sure that certain set of operation do not take place at the same time in two different processes/containers/threads, you can use the global lock.
+
+The global lock can only be used using a `with` statement and specifies the max amount of time you can keep the lock for.
+
+Example:
+
+```python
+from howler.remote.datatypes.lock import Lock
+
+def process_1():
+    with Lock('test', 10):
+        # Do sensitive execution up to 10 seconds
+
+def process_2()
+    with Lock('test', 20):
+        # Do sensitive execution up to 20 seconds
+
+# Process 2 has to wait that process 1 is done before executing
+```
+
+## Sets
+
+Sets are very useful to keep a list of non-duplicated items available to all containers in your cluster.
+
+They support the following methods:
+
+- `add(value1, ...)`: Add one or many values to the set
+- `limited_add(value, limit)`: Add a value only if the number of items already in is smaller then the limit
+- `remove(value)`: Remove a specific value from the set
+- `exist(value)`: Check if a value exists in the set
+- `pop()`: Returns a random value from the set and removes it
+- `pop_all()`: Returns all values from the set and removes them all
+- `random()`: Return a random member of the set but keeps it in the set
+- `members()`: Return all members from the set and leave them there
+- `length()`: Returns the length of the set
+- `delete()`: Deletes the set entirely
+
+Example:
+
+```python
+from howler.remote.datatypes.set import Set
+with Set('test-set') as s:
+    values = ['a', 'b', 1, 2]
+    s.add(*values)
+    print(s.exist('b'))
+    print(s.pop())
+    print(s.length())
+```
+
+Output:
+
+```python
+True
+'a' or 'b' or 1 or 2
+3
+```
+
+## Quota trackers
+
+Quota trackers are used to track user quotas in the system. It has only two functions, one to start tracking an operation and another to end it. If the begin of tracking returns false, the user is over it's quota.
+
+Methods:
+
+- `begin(unique_identifier, max_quota)`: Increment the current quota of the unique identifier if its not already at the maximum, returns `False` if over the quota.
+- `end(unique_identifier)`:  Decrease the current quota of the unique identifier
+
+Example:
+
+```python
+from howler.remote.datatypes.user_quota_tracker import UserQuotaTracker
+uqt = UserQuotaTracker('test-quota')
+
+try:
+    ok = uqt.start('uid', 3):
+    if ok:
+        # Do processing which has quota
+    else:
+        raise Exception('Over your quota...')
+
+finally:
+    if ok:
+        uqt.end('uid')
+```
+
+## Queues
+
+To support distributed processing in a container based environment, this library also support multiple queuing types so the containers can pass messages from one to another.
+
+### Comunication queues
+
+Communication queues are similar to a chat where user connected receive the message but do not know of the message that happened while they were offline.
+
+Available methods:
+
+- `close()`: Stop listening for messages and disconnect from the queue
+- `listen(blocking)`: This is a generator that either wait for message or return None
+- `publish(message)`: Sends a message to the comms queue
+
+Example of message receiver:
+
+```python
+from howler.remote.datatypes.queues.comms import CommsQueue
+
+with CommsQueue('test-comms-queue') as cq:
+    for msg in cq.listen(blocking=True):
+        if msg == "stop":
+            break
+
+        print(message)
+```
+
+Example of message sender:
+
+```python
+from howler.remote.datatypes.queues.comms import CommsQueue
+
+with CommsQueue('test-comms-queue') as cq:
+    # send a message to the receiver
+    cq.publish("This is my message!")
+
+    # tell the receiver to stop
+    cq.publish("stop")
+```
+
+### Named queues (FIFO)
+
+Named queues are essentially First-in First-out (FIFO) queue where messages are processed in order in which they are sent in the queue.
+
+The following methods are available:
+
+- `delete()`: Delete the queue with all it's messages
+- `length()`: Return the lenght of the queue
+- `peek_next()`: Get the next item in the queue without removing it
+- `pop_batch(count)`: Get X amount of items from the queue
+- `pop(blocking)`: Get the next item from the queue. If blocking is True, wait for the next message
+- `push(msg_1, ...)`: Push message(s) to the queue
+- `unpop()`: Put the message back at the head of the FIFO queue
+
+Example:
+
+```python
+from howler.remote.datatypes.queues.named import NamedQueue, select
+with NamedQueue('test-named-queue') as nq:
+    for x in range(5):
+        nq.push(x)
+
+    print(nq.pop_batch(100))
+```
+
+Output:
+
+```python
+[0, 1, 2, 3, 4]
+```
+
+### Priority queues
+
+Priority queues are queues where the message can set its priority on insertion to determine its position in the queue. Message of the same priority will act as FIFO queues.
+
+The following methods are available:
+
+- `count(lowest, highest)`: Count the number of item between the lowest and highest priority (inclusive)
+- `delete()`: Delete the queue with all it's items
+- `length()`: Return the full length of the queue
+- `pop(num)`: Pop the specified number of items form the queue
+- `blocking_pop(timeout, low_priority)`: Pop the next item from the queue and wait for it if the queue is empty (low_priority pops from lower priority items first)
+- `dequeue_range(lower_limit, upper_limit)`: Dequeue a number of elements, within a specified range of scores
+- `push(priority, message)`: Push a message at a given priority
+- `rank(message)`: Returns the priority of a message in the queue
+- `remove(message)`: Removes a message from the queue by its value
+- `unpush(num)`: Pop the specified number of lower priority items from the queue
+
+Example:
+
+```python
+from howler.remote.datatypes.queues.priority import PriorityQueue, length, select
+with PriorityQueue('test-priority-queue') as pq:
+    for x in range(10):
+        pq.push(100, x)
+
+    a_key = pq.push(101, 'a')
+    z_key = pq.push(99, 'z')
+    print(pq.rank(a_key))
+    print(pq.rank(z_key))
+    print(pq.pop())
+```
+
+Output:
+
+```python
+0
+11
+'a'
+```
+
+#### Unique Priority queues
+
+Unique Priority queues function the same way as Priority queues execept that all messages in the queue must be unique so new messages with the same value are not added again.
+
+Example:
+
+```python
+from howler.remote.datatypes.queues.priority import PriorityQueue, length, select
+with PriorityQueue('test-priority-queue') as pq:
+    for x in range(10):
+        pq.push(100, x)
+
+    print(pq.length())
+
+    # These values were already added in the previous loop so they wont be added again.
+    for x in range(10):
+        pq.push(100, x)
+
+    print(pq.length())
+```
+
+Output:
+
+```python
+10
+10
+```

howler/remote/datatypes/__init__.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+import json
+import logging
+import time
+from datetime import datetime
+
+import redis
+from packaging.version import parse
+
+from howler.common import loader
+from howler.odm.models.config import config
+from howler.utils.uid import get_random_id
+
+# Add a version warning if redis python client is < 2.10.0. Older versions
+# have a connection bug that can manifest with the dispatcher.
+if parse(redis.__version__) <= parse("2.10.0"):
+    import warnings
+
+    warnings.warn(
+        "%s works best with redis > 2.10.0. You're running"
+        " redis %s. You should upgrade." % (__name__, redis.__version__)
+    )
+
+
+log = logging.getLogger(f"{loader.APP_NAME}.queue")
+pool: dict[tuple[str, str], redis.BlockingConnectionPool] = {}
+
+
+def now_as_iso():
+    s = datetime.utcfromtimestamp(time.time()).isoformat()
+    return f"{s}Z"
+
+
+def reply_queue_name(prefix=None, suffix=None):
+    if prefix:
+        components = [prefix]
+    else:
+        components = []
+
+    components.append(get_random_id())
+
+    if suffix:
+        components.append(str(suffix))
+
+    return "-".join(components)
+
+
+def retry_call(func, *args, **kw):
+    maximum = 2
+    exponent = -7
+
+    while True:
+        try:
+            ret_val = func(*args, **kw)
+
+            if exponent != -7:
+                log.info("Reconnected to Redis!")
+
+            return ret_val
+        except (redis.ConnectionError, ConnectionResetError) as ce:
+            log.warning(f"No connection to Redis, reconnecting... [{ce}]")
+            time.sleep(2**exponent)
+            exponent = exponent + 1 if exponent < maximum else exponent
+
+
+def get_client(host, port, private):
+    # In case a structure is passed a client as host
+    if isinstance(host, (redis.Redis, redis.StrictRedis)):
+        return host
+
+    if not host or not port:
+        host = host or config.core.redis.nonpersistent.host
+        port = int(port or config.core.redis.nonpersistent.port)
+
+    if private:
+        return redis.StrictRedis(host=host, port=port)
+    else:
+        return redis.StrictRedis(connection_pool=get_pool(host, port))
+
+
+def get_pool(host, port):
+    key = (host, port)
+
+    connection_pool = pool.get(key, None)
+    if not connection_pool:
+        connection_pool = redis.BlockingConnectionPool(host=host, port=port, max_connections=200)
+        pool[key] = connection_pool
+
+    return connection_pool
+
+
+def decode(data):
+    try:
+        return json.loads(data)
+    except ValueError:
+        log.warning("Invalid data on queue: %s", str(data))
+        return None

howler/remote/datatypes/counters.py

@@ -0,0 +1,63 @@
+from typing import Optional
+
+from redis.exceptions import ConnectionError
+
+from howler.remote.datatypes import get_client, now_as_iso, retry_call
+from howler.remote.datatypes.hash import Hash
+
+
+class Counters(object):
+    def __init__(self, prefix="counter", host=None, port=None, track_counters=False):
+        self.c = get_client(host, port, False)
+        self.prefix = prefix
+        if track_counters:
+            self.tracker: Optional[Hash] = Hash("c-tracker-%s" % prefix, host=host, port=port)
+        else:
+            self.tracker = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.delete()
+
+    def inc(self, name, value=1, track_id=None):
+        if self.tracker:
+            self.tracker.add(track_id or name, now_as_iso())
+        return retry_call(self.c.incr, "%s-%s" % (self.prefix, name), value)
+
+    def dec(self, name, value=1, track_id=None):
+        if self.tracker:
+            self.tracker.pop(str(track_id or name))
+        return retry_call(self.c.decr, "%s-%s" % (self.prefix, name), value)
+
+    def get_queues_sizes(self):
+        out = {}
+        for queue in retry_call(self.c.keys, "%s-*" % self.prefix):
+            queue_size = int(retry_call(self.c.get, queue))
+            out[queue] = queue_size
+
+        return {k.decode("utf-8"): v for k, v in out.items()}
+
+    def get_queues(self):
+        return [k.decode("utf-8") for k in retry_call(self.c.keys, "%s-*" % self.prefix)]
+
+    def ready(self):
+        try:
+            self.c.ping()
+        except ConnectionError:
+            return False
+
+        return True
+
+    def reset_queues(self):
+        if self.tracker:
+            self.tracker.delete()
+        for queue in retry_call(self.c.keys, "%s-*" % self.prefix):
+            retry_call(self.c.set, queue, "0")
+
+    def delete(self):
+        if self.tracker:
+            self.tracker.delete()
+        for queue in retry_call(self.c.keys, "%s-*" % self.prefix):
+            retry_call(self.c.delete, queue)

howler/remote/datatypes/events.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import json
+import threading
+from typing import TYPE_CHECKING, Any, Callable, Generic, Optional, TypeVar
+
+from howler.common.logging import get_logger
+from howler.remote.datatypes import get_client, retry_call
+
+if TYPE_CHECKING:
+    from redis import Redis
+
+
+logger = get_logger(__name__)
+
+
+MessageType = TypeVar("MessageType")
+
+
+class EventSender(Generic[MessageType]):
+    def __init__(
+        self,
+        prefix: str,
+        host=None,
+        port=None,
+        private=None,
+        serializer: Callable[[MessageType], str] = json.dumps,
+    ):
+        self.client: Redis[Any] = get_client(host, port, private)
+        self.prefix = prefix.lower()
+        if not self.prefix.endswith("."):
+            self.prefix += "."
+        self.serializer = serializer
+
+    def send(self, name: str, data: MessageType):
+        path = self.prefix + name.lower().lstrip(".")
+        retry_call(self.client.publish, path, self.serializer(data))
+
+
+class EventWatcher(Generic[MessageType]):
+    def __init__(
+        self,
+        host=None,
+        port=None,
+        private=None,
+        deserializer: Callable[[str], MessageType] = json.loads,
+    ):
+        client: Redis[Any] = get_client(host, port, private)
+        self.pubsub = retry_call(client.pubsub)
+        self.worker: Optional[threading.Thread] = None
+        self.deserializer = deserializer
+
+    def register(self, path: str, callback: Callable[[MessageType], None]):
+        def _callback(message: dict[str, Any]):
+            if message["type"] == "pmessage":
+                data = self.deserializer(message.get("data", ""))
+                callback(data)
+
+        self.pubsub.psubscribe(**{path.lower(): _callback})
+
+    def start(self):
+        self.worker = self.pubsub.run_in_thread(0.01, daemon=True)
+
+    def stop(self):
+        if self.worker is not None:
+            self.worker.stop()  # type: ignore