aa-killtracker 0.18.0a2__py3-none-any.whl → 1.0.0a1__py3-none-any.whl

killtracker/managers.py

@@ -15,7 +15,7 @@ from app_utils.logging import LoggerAddTag
 
 from killtracker import __title__
 from killtracker.app_settings import KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS
-from killtracker.core.killmails import Killmail, _KillmailCharacter
+from killtracker.core.zkb import Killmail, _KillmailCharacter
 
 logger = LoggerAddTag(get_extension_logger(__name__), __title__)
 

killtracker/models/trackers.py

@@ -28,11 +28,10 @@ from app_utils.logging import LoggerAddTag
 from killtracker import __title__
 from killtracker.app_settings import KILLTRACKER_KILLMAIL_MAX_AGE_FOR_TRACKER
 from killtracker.constants import EveCategoryId, EveGroupId
-from killtracker.core.discord_messages import DiscordMessage
-from killtracker.core.killmails import Killmail
+from killtracker.core.trackers import create_discord_message_from_killmail
+from killtracker.core.zkb import Killmail
 from killtracker.managers import TrackerManager
-
-from .webhooks import Webhook
+from killtracker.models.webhooks import Webhook
 
 logger = LoggerAddTag(get_extension_logger(__name__), __title__)
 
@@ -886,5 +885,5 @@ class Tracker(models.Model):
 
         Returns the new queue size.
         """
-        message = DiscordMessage.from_killmail(self, killmail, intro_text)
+        message = create_discord_message_from_killmail(self, killmail, intro_text)
         return self.webhook.enqueue_message(message)

killtracker/models/webhooks.py

@@ -2,10 +2,8 @@
 
 from typing import Optional
 
-import dhooks_lite
 from simple_mq import SimpleMQ
 
-from django.core.cache import cache
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
@@ -14,10 +12,9 @@ from app_utils.allianceauth import get_redis_client
 from app_utils.logging import LoggerAddTag
 from app_utils.urls import static_file_absolute_url
 
-from killtracker import APP_NAME, HOMEPAGE_URL, __title__, __version__
+from killtracker import __title__
 from killtracker.app_settings import KILLTRACKER_WEBHOOK_SET_AVATAR
-from killtracker.core.discord_messages import DiscordMessage
-from killtracker.exceptions import WebhookTooManyRequests
+from killtracker.core.discord import DiscordMessage, send_message_to_webhook
 from killtracker.managers import WebhookManager
 
 logger = LoggerAddTag(get_extension_logger(__name__), __title__)
@@ -26,8 +23,6 @@ logger = LoggerAddTag(get_extension_logger(__name__), __title__)
 class Webhook(models.Model):
     """A webhook to receive messages"""
 
-    HTTP_TOO_MANY_REQUESTS = 429
-
     class WebhookType(models.IntegerChoices):
         """A webhook type."""
 
@@ -167,53 +162,9 @@ class Webhook(models.Model):
 
         return q.clear()
 
-    def send_message(self, message: DiscordMessage) -> dhooks_lite.WebhookResponse:
+    def send_message(self, message: DiscordMessage) -> int:
         """Send a message to the webhook."""
-        timeout = cache.ttl(self._blocked_cache_key())  # type: ignore
-        if timeout:
-            raise WebhookTooManyRequests(timeout)
-
-        hook = dhooks_lite.Webhook(
-            url=self.url,
-            user_agent=dhooks_lite.UserAgent(
-                name=APP_NAME, url=HOMEPAGE_URL, version=__version__
-            ),
-        )
-        response = hook.execute(
-            content=message.content,
-            embeds=message.embeds,
-            username=message.username,
-            avatar_url=message.avatar_url,
-            wait_for_response=True,
-            max_retries=0,  # we will handle retries ourselves
-        )
-        logger.debug(
-            "%s: Response from Discord for creating message from killmail %d: %s %s %s",
-            self,
-            message.killmail_id,
-            response.status_code,
-            response.headers,
-            response.content,
-        )
-        if response.status_code == self.HTTP_TOO_MANY_REQUESTS:
-            logger.error(
-                "%s: Received too many requests error from API: %s",
-                self,
-                response.content,
-            )
-            try:
-                retry_after = int(response.headers["Retry-After"]) + 2
-            except (ValueError, KeyError):
-                retry_after = WebhookTooManyRequests.DEFAULT_RESET_AFTER
-            cache.set(
-                key=self._blocked_cache_key(), value="BLOCKED", timeout=retry_after
-            )
-            raise WebhookTooManyRequests(retry_after)
-
-        return response
-
-    def _blocked_cache_key(self) -> str:
-        return f"{__title__}_webhook_{self.pk}_blocked"
+        return send_message_to_webhook(name=self.name, url=self.url, message=message)
 
     @staticmethod
     def create_message_link(name: str, url: str) -> str:

killtracker/signals.py

@@ -8,24 +8,24 @@ from allianceauth.services.hooks import get_extension_logger
 from app_utils.logging import LoggerAddTag
 
 from killtracker import __title__
-from killtracker.core import worker_shutdown
+from killtracker.core import workers
 
 logger = LoggerAddTag(get_extension_logger(__name__), __title__)
 
 
 @signals.worker_ready.connect
 def worker_ready_handler(sender, **kwargs):
-    worker_shutdown.reset(sender.hostname)
+    workers.state_reset(sender.hostname)
     logger.debug("worker_ready: %s", sender.hostname)
 
 
 @signals.worker_shutting_down.connect
 def worker_shutting_down_handler(sender, **kwargs):
-    worker_shutdown.set(sender)
+    workers.state_set(sender)
     logger.debug("worker_shutting_down: %s", sender)
 
 
 @signals.worker_shutdown.connect
 def worker_shutdown_handler(sender, **kwargs):
-    worker_shutdown.reset(sender.hostname)
+    workers.state_reset(sender.hostname)
     logger.debug("worker_shutdown: %s", sender.hostname)

killtracker/tasks.py

@@ -2,7 +2,6 @@
 
 import time
 
-import dhooks_lite
 from celery import Task, chain, shared_task
 
 from django.db import IntegrityError
@@ -21,20 +20,20 @@ from killtracker.app_settings import (
     KILLTRACKER_GENERATE_MESSAGE_MAX_RETRIES,
     KILLTRACKER_GENERATE_MESSAGE_RETRY_COUNTDOWN,
     KILLTRACKER_MAX_KILLMAILS_PER_RUN,
+    KILLTRACKER_MAX_MESSAGES_SENT_PER_RUN,
     KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS,
     KILLTRACKER_RUN_TIMEOUT,
     KILLTRACKER_STORING_KILLMAILS_ENABLED,
     KILLTRACKER_TASK_OBJECTS_CACHE_TIMEOUT,
     KILLTRACKER_TASKS_TIMEOUT,
 )
-from killtracker.core import worker_shutdown
-from killtracker.core.discord_messages import DiscordMessage
-from killtracker.core.killmails import (
-    Killmail,
-    KillmailDoesNotExist,
-    ZKBTooManyRequestsError,
+from killtracker.core import workers
+from killtracker.core.discord import (
+    DiscordMessage,
+    HTTPError,
+    WebhookRateLimitExhausted,
 )
-from killtracker.exceptions import WebhookTooManyRequests
+from killtracker.core.zkb import Killmail, KillmailDoesNotExist, ZKBTooManyRequestsError
 from killtracker.models import EveKillmail, Tracker, Webhook
 
 logger = LoggerAddTag(get_extension_logger(__name__), __title__)
@@ -42,7 +41,8 @@ logger = LoggerAddTag(get_extension_logger(__name__), __title__)
 
 @shared_task(bind=True, base=QueueOnce, timeout=KILLTRACKER_TASKS_TIMEOUT)
 def run_killtracker(self: Task) -> int:
-    """Try to fetch new killmails from ZKB API and start trackers.
+    """Fetches and processes new killmails from ZKB API
+    and returns how many killmails were processed.
 
     This is the main periodic task for running Killtracker.
     """
@@ -64,7 +64,7 @@ def run_killtracker(self: Task) -> int:
         if is_timed_out():
             break
 
-        if worker_shutdown.is_shutting_down(self):
+        if workers.is_shutting_down(self):
             logger.debug("Aborting due to worker shutdown")
             break
 
@@ -203,54 +203,51 @@ def delete_stale_killmails() -> None:
 
 
 @shared_task(
-    bind=True,
-    base=QueueOnce,  # celery_once locks stay intact during retries
-    timeout=KILLTRACKER_TASKS_TIMEOUT,
-    retry_backoff=False,
-    max_retries=None,
+    bind=True, base=QueueOnce, timeout=KILLTRACKER_TASKS_TIMEOUT, max_retries=None
 )
 def send_messages_to_webhook(self: Task, webhook_pk: int) -> None:
-    """Sends all queued messages to given Webhook."""
+    """Sends queued messages to a webhook.
 
-    webhook: Webhook = Webhook.objects.get_cached(
-        pk=webhook_pk,
-        timeout=KILLTRACKER_TASK_OBJECTS_CACHE_TIMEOUT,
-    )
+    Note: This task will retry after processing a set number of messages
+    to avoid running potentially forever.
+    """
+
+    webhook: Webhook = Webhook.objects.get(pk=webhook_pk)
     if not webhook.is_enabled:
         logger.info("%s: Webhook is disabled - aborting", webhook)
         return
 
-    message = webhook.dequeue_message()
-    if not message:
-        logger.debug("%s: No more messages to send for webhook", webhook)
-        return
+    for _ in range(KILLTRACKER_MAX_MESSAGES_SENT_PER_RUN):
+        if workers.is_shutting_down(self):
+            logger.debug("Aborting due to worker shutdown")
+            return
 
-    try:
-        response: dhooks_lite.WebhookResponse = webhook.send_message(message)
-    except WebhookTooManyRequests as ex:
-        webhook.enqueue_message(message)
-        logger.warning(
-            "%s: Too many requests for webhook. Blocked for %s seconds. Aborting.",
-            webhook,
-            ex.retry_after,
-        )
-        return
+        message = webhook.dequeue_message()
+        if not message:
+            logger.debug("%s: No more messages to send for webhook", webhook)
+            break
 
-    if not response.status_ok:
-        webhook.enqueue_message(message, is_error=True)
-        logger.warning(
-            "%s: Failed to send message for Killmail %d to webhook, will retry. "
-            "HTTP status code: %d, response: %s",
-            webhook,
-            message.killmail_id,
-            response.status_code,
-            response.content,
-        )
-    else:
         try:
-            message_id = response.content.get("id")
-        except AttributeError:
-            message_id = "?"
+            message_id = webhook.send_message(message)
+
+        except WebhookRateLimitExhausted as ex:
+            webhook.enqueue_message(message)
+            logger.warning(
+                "%s: Webhook temporarily blocked. Retrying at %s.", webhook, ex.retry_at
+            )
+            raise self.retry(eta=ex.retry_at)
+
+        except HTTPError as ex:
+            webhook.enqueue_message(message, is_error=True)
+            logger.warning(
+                "%s: Failed to send message for Killmail %d to webhook, will retry. "
+                "HTTP status code: %d",
+                webhook,
+                message.killmail_id,
+                ex.status_code,
+            )
+            continue
+
         logger.info(
             "%s: Discord message %s created for killmail %d",
             webhook,
@@ -258,7 +255,8 @@ def send_messages_to_webhook(self: Task, webhook_pk: int) -> None:
             message.killmail_id,
         )
 
-    raise self.retry(countdown=KILLTRACKER_DISCORD_SEND_DELAY)
+    if webhook.messages_queued() > 0:
+        raise self.retry(countdown=KILLTRACKER_DISCORD_SEND_DELAY)
 
 
 @shared_task(timeout=KILLTRACKER_TASKS_TIMEOUT)

killtracker/tests/core/test_discord.py

@@ -0,0 +1,184 @@
+import datetime as dt
+from unittest.mock import patch
+
+import dhooks_lite
+import requests_mock
+
+from django.utils.timezone import now
+
+from app_utils.testing import NoSocketsTestCase
+
+from killtracker.core.discord import (
+    DiscordMessage,
+    HTTPError,
+    WebhookRateLimitExhausted,
+    _make_key_last_request,
+    _make_key_retry_at,
+    send_message_to_webhook,
+)
+from killtracker.tests.utils import CacheFake
+
+MODULE_PATH = "killtracker.core.discord"
+
+
+class TestDiscordMessage(NoSocketsTestCase):
+    def test_can_create(self):
+        o = DiscordMessage(content="content")
+        self.assertEqual(o.content, "content")
+
+    def test_should_raise_exception_when_invalid(self):
+        with self.assertRaises(ValueError):
+            DiscordMessage(username="user")
+
+    def test_can_convert_to_and_from_json_1(self):
+        o1 = DiscordMessage(
+            content="content",
+        )
+        s = o1.to_json()
+        o2 = DiscordMessage.from_json(s)
+        self.assertEqual(o1, o2)
+
+    def test_can_convert_to_and_from_json_2(self):
+        o1 = DiscordMessage(
+            avatar_url="avatar_url",
+            content="content",
+            embeds=[dhooks_lite.Embed(description="description")],
+            killmail_id=42,
+            username="username",
+        )
+        s = o1.to_json()
+        o2 = DiscordMessage.from_json(s)
+        self.assertEqual(o1, o2)
+
+
+@requests_mock.Mocker()
+@patch(MODULE_PATH + ".cache", new_callable=CacheFake)
+class TestWebhookSendMessage(NoSocketsTestCase):
+    def setUp(self) -> None:
+        self.name = "webhook"
+        self.message = DiscordMessage(content="Test message")
+        self.url = "https://webhook.example.com/1234"
+        self.message_api = {
+            "name": "test webhook",
+            "type": 1,
+            "channel_id": "199737254929760256",
+            "token": "3d89bb7572e0fb30d8128367b3b1b44fecd1726de135cbe28a41f8b2f777c372ba2939e72279b94526ff5d1bd4358d65cf11",
+            "avatar": None,
+            "guild_id": "199737254929760256",
+            "id": "223704706495545344",
+            "application_id": None,
+            "user": {
+                "username": "test",
+                "discriminator": "7479",
+                "id": "190320984123768832",
+                "avatar": "b004ec1740a63ca06ae2e14c5cee11f3",
+                "public_flags": 131328,
+            },
+        }
+
+    def test_when_send_ok_returns_true(self, requests_mocker, mock_cache):
+        # given
+        requests_mocker.register_uri(
+            "POST", self.url, status_code=200, json=self.message_api
+        )
+        # when
+        got = send_message_to_webhook(
+            name=self.name, url=self.url, message=self.message
+        )
+        # then
+        self.assertEqual(got, 223704706495545344)
+        self.assertTrue(requests_mocker.called)
+
+    def test_should_ignore_invalid_key_for_last_request(
+        self, requests_mocker, mock_cache
+    ):
+        # given
+        mock_cache.set(_make_key_last_request(self.url), "invalid")
+        requests_mocker.register_uri(
+            "POST", self.url, status_code=200, json=self.message_api
+        )
+        # when
+        got = send_message_to_webhook(
+            name=self.name, url=self.url, message=self.message
+        )
+        # then
+        self.assertEqual(got, 223704706495545344)
+        self.assertTrue(requests_mocker.called)
+
+    def test_should_ignore_invalid_key_for_retry_at(self, requests_mocker, mock_cache):
+        # given
+        mock_cache.set(_make_key_retry_at(self.url), "invalid")
+        requests_mocker.register_uri(
+            "POST", self.url, status_code=200, json=self.message_api
+        )
+        # when
+        got = send_message_to_webhook(
+            name=self.name, url=self.url, message=self.message
+        )
+        # then
+        self.assertEqual(got, 223704706495545344)
+        self.assertTrue(requests_mocker.called)
+
+    def test_when_send_not_ok_raise_error(self, requests_mocker, mock_cache):
+        # given
+        requests_mocker.register_uri("POST", self.url, status_code=404)
+        # when
+        with self.assertRaises(HTTPError) as ctx:
+            send_message_to_webhook(name=self.name, url=self.url, message=self.message)
+        # then
+        self.assertEqual(ctx.exception.status_code, 404)
+        self.assertTrue(requests_mocker.called)
+
+    def test_raise_too_many_requests_when_received_from_api(
+        self, requests_mocker, mock_cache
+    ):
+        # given
+        requests_mocker.register_uri(
+            "POST",
+            self.url,
+            status_code=429,
+            json={
+                "global": False,
+                "message": "You are being rate limited.",
+                "retry_after": 2000,
+            },
+            headers={
+                "x-ratelimit-remaining": "5",
+                "x-ratelimit-reset-after": "60",
+                "Retry-After": "2000",
+            },
+        )
+        # when/then
+        with self.assertRaises(WebhookRateLimitExhausted) as ctx:
+            send_message_to_webhook(name=self.name, url=self.url, message=self.message)
+
+        self.assertTrue(ctx.exception.retry_at)
+
+    def test_too_many_requests_no_retry_value(self, requests_mocker, mock_cache):
+        # given
+        requests_mocker.register_uri(
+            "POST",
+            self.url,
+            status_code=429,
+            headers={
+                "x-ratelimit-remaining": "5",
+                "x-ratelimit-reset-after": "60",
+            },
+        )
+        # when/then
+        with self.assertRaises(WebhookRateLimitExhausted) as ctx:
+            send_message_to_webhook(name=self.name, url=self.url, message=self.message)
+
+        self.assertTrue(ctx.exception.retry_at)
+
+    def test_should_reraise_exception_when_not_expired(
+        self, requests_mocker, mock_cache
+    ):
+        # given
+        key = _make_key_retry_at(self.url)
+        mock_cache.set(key, now() + dt.timedelta(hours=1))
+        # when
+        with self.assertRaises(WebhookRateLimitExhausted) as ctx:
+            send_message_to_webhook(name=self.name, url=self.url, message=self.message)
+        # then
+        self.assertTrue(ctx.exception.retry_at)

killtracker/tests/core/test_helpers.py

@@ -0,0 +1,23 @@
+import datetime as dt
+from typing import Any, NamedTuple, Optional
+from unittest import TestCase
+
+from killtracker.core.helpers import datetime_or_none
+
+
+class TestDatetimeOrNone(TestCase):
+    def test_should_return_value(self):
+        class Case(NamedTuple):
+            value: Any
+            want: Optional[dt.datetime]
+
+        now = dt.datetime.now()
+        cases = [
+            Case(now, now),
+            Case("abc", None),
+            Case(42, None),
+            Case(None, None),
+        ]
+        for tc in cases:
+            got = datetime_or_none(tc.value)
+            self.assertIs(got, tc.want)

killtracker/tests/core/{test_discord_messages_1.py → test_tracker_1.py}

@@ -2,7 +2,10 @@ import dhooks_lite
 
 from app_utils.testing import NoSocketsTestCase
 
-from killtracker.core import discord_messages
+from killtracker.core.trackers import (
+    _create_embed,
+    create_discord_message_from_killmail,
+)
 from killtracker.tests.testdata.factories import (
     EveEntityVariant,
     KillmailFactory,
@@ -25,7 +28,7 @@ class TestCreateEmbed(NoSocketsTestCase):
         tracker = TrackerFactory()
         killmail = KillmailFactory()
         # when
-        embed = discord_messages._create_embed(tracker, killmail)
+        embed = _create_embed(tracker, killmail)
         # then
         self.assertIsInstance(embed, dhooks_lite.Embed)
 
@@ -34,7 +37,7 @@ class TestCreateEmbed(NoSocketsTestCase):
         tracker = TrackerFactory()
         killmail = KillmailFactory(zkb__total_value=None)
         # when
-        embed = discord_messages._create_embed(tracker, killmail)
+        embed = _create_embed(tracker, killmail)
         # then
         self.assertIsInstance(embed, dhooks_lite.Embed)
 
@@ -43,7 +46,7 @@ class TestCreateEmbed(NoSocketsTestCase):
         tracker = TrackerFactory()
         killmail = KillmailFactory(victim__alliance_id=None)
         # when
-        embed = discord_messages._create_embed(tracker, killmail)
+        embed = _create_embed(tracker, killmail)
         # then
         self.assertIsInstance(embed, dhooks_lite.Embed)
 
@@ -54,7 +57,7 @@ class TestCreateEmbed(NoSocketsTestCase):
             victim__alliance_id=None, victim__corporation_id=None
         )
         # when
-        embed = discord_messages._create_embed(tracker, killmail)
+        embed = _create_embed(tracker, killmail)
         # then
         self.assertIsInstance(embed, dhooks_lite.Embed)
 
@@ -64,7 +67,7 @@ class TestCreateEmbed(NoSocketsTestCase):
         killmail = KillmailFactory()
         killmail.attackers.remove(killmail.attacker_final_blow())
         # when
-        embed = discord_messages._create_embed(tracker, killmail)
+        embed = _create_embed(tracker, killmail)
         # then
         self.assertIsInstance(embed, dhooks_lite.Embed)
 
@@ -73,7 +76,7 @@ class TestCreateEmbed(NoSocketsTestCase):
         tracker = TrackerFactory()
         killmail = KillmailFactory().clone_with_tracker_info(tracker.pk)
         # when
-        embed = discord_messages._create_embed(tracker, killmail)
+        embed = _create_embed(tracker, killmail)
         # then
         self.assertIsInstance(embed, dhooks_lite.Embed)
 
@@ -85,41 +88,11 @@ class TestCreateEmbed(NoSocketsTestCase):
             tracker.pk, jumps=3, distance=3.5, matching_ship_type_ids=[ship_type.id]
         )
         # when
-        embed = discord_messages._create_embed(tracker, killmail)
+        embed = _create_embed(tracker, killmail)
         # then
         self.assertIsInstance(embed, dhooks_lite.Embed)
 
 
-class TestDiscordMessage(NoSocketsTestCase):
-    def test_can_create(self):
-        o = discord_messages.DiscordMessage(content="content")
-        self.assertEqual(o.content, "content")
-
-    def test_should_raise_exception_when_invalid(self):
-        with self.assertRaises(ValueError):
-            discord_messages.DiscordMessage(username="user")
-
-    def test_can_convert_to_and_from_json_1(self):
-        o1 = discord_messages.DiscordMessage(
-            content="content",
-        )
-        s = o1.to_json()
-        o2 = discord_messages.DiscordMessage.from_json(s)
-        self.assertEqual(o1, o2)
-
-    def test_can_convert_to_and_from_json_2(self):
-        o1 = discord_messages.DiscordMessage(
-            avatar_url="avatar_url",
-            content="content",
-            embeds=[dhooks_lite.Embed(description="description")],
-            killmail_id=42,
-            username="username",
-        )
-        s = o1.to_json()
-        o2 = discord_messages.DiscordMessage.from_json(s)
-        self.assertEqual(o1, o2)
-
-
 class TestDiscordMessageFromKillmail(NoSocketsTestCase):
     @classmethod
     def setUpClass(cls):
@@ -132,6 +105,6 @@ class TestDiscordMessageFromKillmail(NoSocketsTestCase):
         tracker = TrackerFactory()
         killmail = KillmailFactory()
         # when
-        m = discord_messages.DiscordMessage.from_killmail(tracker, killmail)
+        m = create_discord_message_from_killmail(tracker, killmail)
         # then
         self.assertIsInstance(m.embeds[0], dhooks_lite.Embed)

killtracker/tests/core/{test_discord_messages_2.py → test_tracker_2.py}

@@ -8,16 +8,16 @@ from django.test import TestCase
 
 from app_utils.django import app_labels
 
-from killtracker.core.killmails import Killmail
+from killtracker.core.zkb import Killmail
 from killtracker.models import Tracker
 from killtracker.tests.testdata.factories import TrackerFactory
 from killtracker.tests.testdata.helpers import LoadTestDataMixin, load_killmail
 
-DISCORD_MESSAGES_PATH = "killtracker.core.discord_messages"
+MODULE_PATH = "killtracker.core.trackers"
 
 if "discord" in app_labels():
 
-    @patch(DISCORD_MESSAGES_PATH + "._import_discord_user")
+    @patch(MODULE_PATH + "._import_discord_user")
     class TestGroupPings(LoadTestDataMixin, TestCase):
         @classmethod
         def setUpClass(cls):