aa-killtracker 0.16.0__py3-none-any.whl → 0.18.0a2__py3-none-any.whl

killtracker/core/killmails.py

@@ -2,36 +2,35 @@
 
 # pylint: disable = redefined-builtin
 
+import datetime as dt
 import json
 from copy import deepcopy
 from dataclasses import asdict, dataclass
-from datetime import datetime
 from http import HTTPStatus
+from time import sleep
 from typing import List, Optional, Set
 from urllib.parse import quote_plus
 
 import requests
 from dacite import DaciteError, from_dict
-from redis.exceptions import LockError
 from simplejson.errors import JSONDecodeError
 
-from django.conf import settings
 from django.core.cache import cache
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.dateparse import parse_datetime
+from django.utils.timezone import now
 from eveuniverse.models import EveType
 
 from allianceauth.services.hooks import get_extension_logger
-from app_utils.allianceauth import get_redis_client
 from app_utils.json import JSONDateTimeDecoder, JSONDateTimeEncoder
 from app_utils.logging import LoggerAddTag
 
 from killtracker import USER_AGENT_TEXT, __title__
 from killtracker.app_settings import (
     KILLTRACKER_QUEUE_ID,
-    KILLTRACKER_REDISQ_LOCK_TIMEOUT,
     KILLTRACKER_REDISQ_TTW,
     KILLTRACKER_STORAGE_KILLMAILS_LIFETIME,
+    KILLTRACKER_ZKB_REQUEST_DELAY,
 )
 from killtracker.exceptions import KillmailDoesNotExist
 from killtracker.providers import esi
@@ -42,6 +41,7 @@ ZKB_REDISQ_URL = "https://zkillredisq.stream/listen.php"
 ZKB_API_URL = "https://zkillboard.com/api/"
 ZKB_KILLMAIL_BASEURL = "https://zkillboard.com/kill/"
 REQUESTS_TIMEOUT = (5, 30)
+DEFAULT_429_TIMEOUT = 10
 
 MAIN_MINIMUM_COUNT = 2
 MAIN_MINIMUM_SHARE = 0.25
@@ -49,6 +49,14 @@ MAIN_MINIMUM_SHARE = 0.25
 # TODO: Factor out logic for accessing the API to another module
 
 
+class ZKBTooManyRequestsError(Exception):
+    """ZKB RedisQ API has returned 429 Too Many Requests HTTP status code."""
+
+    def __init__(self, retry_at: dt.datetime, is_original: bool = True):
+        self.retry_at = retry_at
+        self.is_original = is_original
+
+
 @dataclass
 class _KillmailBase:
     """Base class for all Killmail."""
@@ -160,7 +168,7 @@ class Killmail(_KillmailBase):
     _STORAGE_BASE_KEY = "killtracker_storage_killmail_"
 
     id: int
-    time: datetime
+    time: dt.datetime
     victim: KillmailVictim
     attackers: List[KillmailAttacker]
     position: KillmailPosition
@@ -363,7 +371,10 @@ class Killmail(_KillmailBase):
 
     @classmethod
     def get(cls, id: int) -> "Killmail":
-        """Fetch a killmail from temporary storage."""
+        """Fetch a killmail from temporary storage.
+
+        Raises KillmailDoesNotExist if killmail does not exit.
+        """
         data = cache.get(key=cls._storage_key(id))
         if not data:
             raise KillmailDoesNotExist(
@@ -391,9 +402,14 @@ class Killmail(_KillmailBase):
 
     @classmethod
     def create_from_zkb_redisq(cls) -> Optional["Killmail"]:
-        """Fetches and returns a killmail from ZKB.
+        """Fetches and returns a killmail from ZKB REDISQ API.
+
+        Will automatically wait for a free rate limit slot if needed.
+        Will re-raise TooManyRequests if a recent 429 timeout is not yet expired.
 
-        Returns None if no killmail is received.
+        This method is not thread safe.
+
+        Returns None if no killmail was received.
         """
         if not KILLTRACKER_QUEUE_ID:
             raise ImproperlyConfigured(
@@ -403,51 +419,85 @@ class Killmail(_KillmailBase):
         if "," in KILLTRACKER_QUEUE_ID:
             raise ImproperlyConfigured("A queue ID must not contains commas.")
 
-        redis = get_redis_client()
-        params = {
-            "queueID": quote_plus(KILLTRACKER_QUEUE_ID),
-            "ttw": KILLTRACKER_REDISQ_TTW,
-        }
-        try:
-            logger.info("Trying to fetch killmail from ZKB RedisQ...")
-            with redis.lock(
-                cls.lock_key(), blocking_timeout=KILLTRACKER_REDISQ_LOCK_TIMEOUT
-            ):
-                response = requests.get(
-                    ZKB_REDISQ_URL,
-                    params=params,
-                    timeout=REQUESTS_TIMEOUT,
-                    headers={"User-Agent": USER_AGENT_TEXT},
-                )
-        except LockError:
+        key_retry_at = "killtracker-retry-at"
+        if (v := cache.get(key_retry_at)) is not None:
+            try:
+                retry_at = dt.datetime.fromisoformat(v)
+            except (TypeError, ValueError):
+                cache.delete(key_retry_at)
+                logger.warning("unable to parse timestamp of retry at")
+                return None
+
+            if retry_at > now():
+                raise ZKBTooManyRequestsError(retry_at=retry_at, is_original=False)
+
+        key_last_request = "killtracker-last-request"
+        if (v := cache.get(key_last_request)) is not None:
+            try:
+                last_request = dt.datetime.fromisoformat(v)
+            except (TypeError, ValueError):
+                cache.delete(key_last_request)
+                logger.warning("unable to parse timestamp of last request")
+                last_request = now()
+
+            next_slot = last_request + dt.timedelta(
+                milliseconds=KILLTRACKER_ZKB_REQUEST_DELAY
+            )
+            seconds = (next_slot - now()).total_seconds()
+            if seconds > 0:
+                logger.debug("Waiting %f seconds to for next free slot", seconds)
+                sleep(seconds)
+
+        response = requests.get(
+            ZKB_REDISQ_URL,
+            params={
+                "queueID": quote_plus(KILLTRACKER_QUEUE_ID),
+                "ttw": KILLTRACKER_REDISQ_TTW,
+            },
+            timeout=REQUESTS_TIMEOUT,
+            headers={"User-Agent": USER_AGENT_TEXT},
+        )
+        cache.set(key_last_request, dt.datetime.isoformat(now()), timeout=30)
+        logger.debug(
+            "Response from ZKB API: %d %s %s",
+            response.status_code,
+            response.headers,
+            response.text,
+        )
+
+        if not response.ok:
             logger.warning(
-                "Failed to acquire lock for atomic access to RedisQ.",
-                exc_info=settings.DEBUG,  # provide details in DEBUG mode
+                "ZKB API returned error: %d %s", response.status_code, response.text
             )
-            return None
+            if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
+                try:
+                    retry_after = int(response.headers["Retry-After"])
+                except KeyError:
+                    retry_after = DEFAULT_429_TIMEOUT
+                retry_at = now() + dt.timedelta(seconds=retry_after)
+                cache.set(key_retry_at, retry_at.isoformat(), timeout=retry_after + 60)
+                raise ZKBTooManyRequestsError(retry_at=retry_at, is_original=True)
 
-        if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
-            logger.error("429 Client Error: Too many requests: %s", response.text)
             return None
 
-        response.raise_for_status()
-
         try:
             data = response.json()
         except JSONDecodeError:
-            logger.error("Error from ZKB API:\n%s", response.text)
+            logger.error("Error parsing ZKB API response:\n%s", response.text)
             return None
 
-        if data:
-            logger.debug("data:\n%s", data)
+        if not data or "package" not in data or not data["package"]:
+            logger.info("ZKB did not return a killmail")
+            return None
 
-        if data and "package" in data and data["package"]:
-            logger.info("Received a killmail from ZKB RedisQ")
-            package_data = data["package"]
-            return cls._create_from_dict(package_data)
+        package_data = data["package"]
+        km = cls._create_from_dict(package_data)
+        if km is not None:
+            logger.info("ZKB returned killmail %d", km.id)
+        else:
+            logger.info("Failed to parse killmail from ZKB")
 
-        logger.debug("Did not received a killmail from ZKB RedisQ")
-        return None
+        return km
 
     @classmethod
     def create_from_zkb_api(cls, killmail_id: int) -> Optional["Killmail"]:
@@ -601,20 +651,3 @@ class Killmail(_KillmailBase):
                     params[prop] = zkb_data[prop]
 
         return KillmailZkb(**params)
-
-    @staticmethod
-    def lock_key() -> str:
-        """Key used for lock operation on Redis."""
-        return f"{__title__.upper()}_REDISQ_LOCK"
-
-    @classmethod
-    def reset_lock_key(cls):
-        """Delete lock key if it exists.
-
-        It can happen that a lock key is not cleaned up
-        and then prevents this class from ever acquiring a lock again.
-        To prevent this we are deleting the lock key at system start.
-        """
-        redis = get_redis_client()
-        if redis.delete(cls.lock_key()) > 0:
-            logger.warning("A stuck lock key was cleared.")

killtracker/core/worker_shutdown.py

@@ -0,0 +1,47 @@
+"""Module worker_shutdown allows tasks to find out
+whether their worker is currently shutting down.
+
+This enables long running tasks to abort early,
+which helps to speed up a warm worker shutdown.
+"""
+
+from celery import Task
+
+from django.core.cache import cache
+
+from allianceauth.services.hooks import get_extension_logger
+from app_utils.logging import LoggerAddTag
+
+from killtracker import __title__
+
+_TIMEOUT_SECONDS = 120
+
+logger = LoggerAddTag(get_extension_logger(__name__), __title__)
+
+
+def reset(hostname: str) -> None:
+    """Resets the shutting down state for a worker."""
+    cache.delete(_make_key(hostname))
+
+
+def set(hostname: str) -> None:
+    """Sets a worker into the shutting down state."""
+    cache.set(_make_key(hostname), "shutting down", timeout=_TIMEOUT_SECONDS)
+
+
+def is_shutting_down(task: Task) -> bool:
+    """Reports whether the worker of a celery task is currently shutting down."""
+    try:
+        hostname = str(task.request.hostname)
+    except (AttributeError, TypeError, ValueError):
+        logger.warning("Failed to retrieve hostname: %s", task)
+        return False
+
+    if cache.get(_make_key(hostname)) is None:
+        return False
+
+    return True
+
+
+def _make_key(hostname: str) -> str:
+    return f"killtracker-worker-shutting-down-{hostname}"

killtracker/forms.py

@@ -7,7 +7,7 @@ from django.core.exceptions import ValidationError
 from django.forms.widgets import TextInput
 from django.utils.translation import gettext_lazy as _
 
-from .models import Tracker
+from killtracker.models import Tracker
 
 
 def field_nice_display(name: str) -> str:

killtracker/managers.py

@@ -13,9 +13,9 @@ from allianceauth.services.hooks import get_extension_logger
 from app_utils.caching import ObjectCacheMixin
 from app_utils.logging import LoggerAddTag
 
-from . import __title__
-from .app_settings import KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS
-from .core.killmails import Killmail, _KillmailCharacter
+from killtracker import __title__
+from killtracker.app_settings import KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS
+from killtracker.core.killmails import Killmail, _KillmailCharacter
 
 logger = LoggerAddTag(get_extension_logger(__name__), __title__)
 

killtracker/models/trackers.py

@@ -28,6 +28,7 @@ from app_utils.logging import LoggerAddTag
 from killtracker import __title__
 from killtracker.app_settings import KILLTRACKER_KILLMAIL_MAX_AGE_FOR_TRACKER
 from killtracker.constants import EveCategoryId, EveGroupId
+from killtracker.core.discord_messages import DiscordMessage
 from killtracker.core.killmails import Killmail
 from killtracker.managers import TrackerManager
 
@@ -881,12 +882,9 @@ class Tracker(models.Model):
     def generate_killmail_message(
         self, killmail: Killmail, intro_text: Optional[str] = None
     ) -> int:
-        """generate a message from given killmail and enqueue for later sending
+        """Generate a message from given killmail and enqueue for later sending.
 
-        returns new queue size
+        Returns the new queue size.
         """
-        from killtracker.core import discord_messages
-
-        content = discord_messages.create_content(self, intro_text)
-        embed = discord_messages.create_embed(self, killmail)
-        return self.webhook.enqueue_message(content=content, embeds=[embed])
+        message = DiscordMessage.from_killmail(self, killmail, intro_text)
+        return self.webhook.enqueue_message(message)

killtracker/models/webhooks.py

@@ -1,7 +1,6 @@
 """Webhooks models for killtracker."""
 
-import json
-from typing import List, Optional
+from typing import Optional
 
 import dhooks_lite
 from simple_mq import SimpleMQ
@@ -12,12 +11,12 @@ from django.utils.translation import gettext_lazy as _
 
 from allianceauth.services.hooks import get_extension_logger
 from app_utils.allianceauth import get_redis_client
-from app_utils.json import JSONDateTimeDecoder, JSONDateTimeEncoder
 from app_utils.logging import LoggerAddTag
 from app_utils.urls import static_file_absolute_url
 
 from killtracker import APP_NAME, HOMEPAGE_URL, __title__, __version__
 from killtracker.app_settings import KILLTRACKER_WEBHOOK_SET_AVATAR
+from killtracker.core.discord_messages import DiscordMessage
 from killtracker.exceptions import WebhookTooManyRequests
 from killtracker.managers import WebhookManager
 
@@ -63,8 +62,8 @@ class Webhook(models.Model):
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
-        self.main_queue = self._create_queue("main")
-        self.error_queue = self._create_queue("error")
+        self._main_queue = self._create_queue("main")
+        self._error_queue = self._create_queue("error")
 
     def __str__(self) -> str:
         return self.name
@@ -78,8 +77,8 @@ class Webhook(models.Model):
         # method to avoid modifying the original state.
         state = self.__dict__.copy()
         # Remove the unpicklable entries.
-        del state["main_queue"]
-        del state["error_queue"]
+        del state["_main_queue"]
+        del state["_error_queue"]
         return state
 
     def __setstate__(self, state):
@@ -87,15 +86,15 @@ class Webhook(models.Model):
         self.__dict__.update(state)
         # Restore the previously opened file's state. To do so, we need to
         # reopen it and read from it until the line count is restored.
-        self.main_queue = self._create_queue("main")
-        self.error_queue = self._create_queue("error")
+        self._main_queue = self._create_queue("main")
+        self._error_queue = self._create_queue("error")
 
     def save(self, *args, **kwargs):
         is_new = self.id is None  # type: ignore
         super().save(*args, **kwargs)
         if is_new:
-            self.main_queue = self._create_queue("main")
-            self.error_queue = self._create_queue("error")
+            self._main_queue = self._create_queue("main")
+            self._error_queue = self._create_queue("error")
 
     def _create_queue(self, suffix: str) -> Optional[SimpleMQ]:
         redis_client = get_redis_client()
@@ -110,94 +109,70 @@ class Webhook(models.Model):
         returns number of moved messages.
         """
         counter = 0
-        if self.error_queue and self.main_queue:
+        if self._error_queue and self._main_queue:
             while True:
-                message = self.error_queue.dequeue()
+                message = self._error_queue.dequeue()
                 if message is None:
                     break
 
-                self.main_queue.enqueue(message)
+                self._main_queue.enqueue(message)
                 counter += 1
 
         return counter
 
-    def enqueue_message(
-        self,
-        content: Optional[str] = None,
-        embeds: Optional[List[dhooks_lite.Embed]] = None,
-        tts: Optional[bool] = None,
-        username: Optional[str] = None,
-        avatar_url: Optional[str] = None,
-    ) -> int:
-        """Enqueues a message to be send with this webhook"""
-        if not self.main_queue:
+    def enqueue_message(self, message: DiscordMessage, is_error: bool = False) -> int:
+        """Enqueues a discord message to be send with this webhook.
+
+        Returns the updated number of messages in the main queue.
+        """
+        q = self._error_queue if is_error else self._main_queue
+
+        if not q:
             return 0
 
-        username = __title__ if KILLTRACKER_WEBHOOK_SET_AVATAR else username
-        brand_url = static_file_absolute_url("killtracker/killtracker_logo.png")
-        avatar_url = brand_url if KILLTRACKER_WEBHOOK_SET_AVATAR else avatar_url
-        return self.main_queue.enqueue(
-            self._discord_message_asjson(
-                content=content,
-                embeds=embeds,
-                tts=tts,
-                username=username,
-                avatar_url=avatar_url,
-            )
-        )
+        if KILLTRACKER_WEBHOOK_SET_AVATAR:
+            message.username = __title__
 
-    @staticmethod
-    def _discord_message_asjson(
-        content: Optional[str] = None,
-        embeds: Optional[List[dhooks_lite.Embed]] = None,
-        tts: Optional[bool] = None,
-        username: Optional[str] = None,
-        avatar_url: Optional[str] = None,
-    ) -> str:
-        """Converts a Discord message to JSON and returns it
-
-        Raises ValueError if message is incomplete
-        """
-        if not content and not embeds:
-            raise ValueError("Message must have content or embeds to be valid")
-
-        if embeds:
-            embeds_list = [obj.asdict() for obj in embeds]
-        else:
-            embeds_list = None
-
-        message = {}
-        if content:
-            message["content"] = content
-        if embeds_list:
-            message["embeds"] = embeds_list
-        if tts:
-            message["tts"] = tts
-        if username:
-            message["username"] = username
-        if avatar_url:
-            message["avatar_url"] = avatar_url
-
-        return json.dumps(message, cls=JSONDateTimeEncoder)
-
-    def send_message_to_webhook(self, message_json: str) -> dhooks_lite.WebhookResponse:
-        """Send given message to webhook
-
-        Params
-            message_json: Discord message encoded in JSON
+        if KILLTRACKER_WEBHOOK_SET_AVATAR:
+            brand_url = static_file_absolute_url("killtracker/killtracker_logo.png")
+            message.avatar_url = brand_url
+
+        return q.enqueue(message.to_json())
+
+    def dequeue_message(self, is_error: bool = False) -> Optional[DiscordMessage]:
+        """Dequeues a message from the main queue and return it.
+
+        Returns None if the queue is empty.
         """
+        q = self._error_queue if is_error else self._main_queue
+        s = q.dequeue()
+        if not s:
+            return None
+
+        return DiscordMessage.from_json(s)
+
+    def messages_queued(self, is_error: bool = False) -> int:
+        """Returns how many message are currently in the queue."""
+        q = self._error_queue if is_error else self._main_queue
+        if not q:
+            return 0
+
+        return q.size()
+
+    def delete_queued_messages(self, is_error: bool = False) -> int:
+        """Deletes all messages in a queue and returns how many messages where deleted."""
+        q = self._error_queue if is_error else self._main_queue
+        if not q:
+            return 0
+
+        return q.clear()
+
+    def send_message(self, message: DiscordMessage) -> dhooks_lite.WebhookResponse:
+        """Send a message to the webhook."""
         timeout = cache.ttl(self._blocked_cache_key())  # type: ignore
         if timeout:
             raise WebhookTooManyRequests(timeout)
 
-        message = json.loads(message_json, cls=JSONDateTimeDecoder)
-        if message.get("embeds"):
-            embeds = [
-                dhooks_lite.Embed.from_dict(embed_dict)
-                for embed_dict in message.get("embeds")
-            ]
-        else:
-            embeds = None
         hook = dhooks_lite.Webhook(
             url=self.url,
             user_agent=dhooks_lite.UserAgent(
@@ -205,16 +180,21 @@ class Webhook(models.Model):
             ),
         )
         response = hook.execute(
-            content=message.get("content"),
-            embeds=embeds,
-            username=message.get("username"),
-            avatar_url=message.get("avatar_url"),
+            content=message.content,
+            embeds=message.embeds,
+            username=message.username,
+            avatar_url=message.avatar_url,
             wait_for_response=True,
             max_retries=0,  # we will handle retries ourselves
         )
-        logger.debug("headers: %s", response.headers)
-        logger.debug("status_code: %s", response.status_code)
-        logger.debug("content: %s", response.content)
+        logger.debug(
+            "%s: Response from Discord for creating message from killmail %d: %s %s %s",
+            self,
+            message.killmail_id,
+            response.status_code,
+            response.headers,
+            response.content,
+        )
         if response.status_code == self.HTTP_TOO_MANY_REQUESTS:
             logger.error(
                 "%s: Received too many requests error from API: %s",
@@ -229,6 +209,7 @@ class Webhook(models.Model):
                 key=self._blocked_cache_key(), value="BLOCKED", timeout=retry_after
             )
             raise WebhookTooManyRequests(retry_after)
+
         return response
 
     def _blocked_cache_key(self) -> str:

killtracker/providers.py

@@ -5,7 +5,7 @@ from esi.clients import EsiClientProvider
 from allianceauth.services.hooks import get_extension_logger
 from app_utils.logging import LoggerAddTag
 
-from . import USER_AGENT_TEXT, __title__
+from killtracker import USER_AGENT_TEXT, __title__
 
 logger = LoggerAddTag(get_extension_logger(__name__), __title__)
 

killtracker/signals.py

@@ -0,0 +1,31 @@
+"""Module signals connects to celery signals."""
+
+# pylint: disable=missing-function-docstring
+
+from celery import signals
+
+from allianceauth.services.hooks import get_extension_logger
+from app_utils.logging import LoggerAddTag
+
+from killtracker import __title__
+from killtracker.core import worker_shutdown
+
+logger = LoggerAddTag(get_extension_logger(__name__), __title__)
+
+
+@signals.worker_ready.connect
+def worker_ready_handler(sender, **kwargs):
+    worker_shutdown.reset(sender.hostname)
+    logger.debug("worker_ready: %s", sender.hostname)
+
+
+@signals.worker_shutting_down.connect
+def worker_shutting_down_handler(sender, **kwargs):
+    worker_shutdown.set(sender)
+    logger.debug("worker_shutting_down: %s", sender)
+
+
+@signals.worker_shutdown.connect
+def worker_shutdown_handler(sender, **kwargs):
+    worker_shutdown.reset(sender.hostname)
+    logger.debug("worker_shutdown: %s", sender.hostname)