aa-killtracker 0.17.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {aa_killtracker-0.17.0.dist-info → aa_killtracker-1.0.0.dist-info}/METADATA +7 -8
  2. {aa_killtracker-0.17.0.dist-info → aa_killtracker-1.0.0.dist-info}/RECORD +36 -29
  3. killtracker/__init__.py +1 -1
  4. killtracker/admin.py +13 -8
  5. killtracker/app_settings.py +20 -10
  6. killtracker/apps.py +2 -4
  7. killtracker/core/discord.py +162 -0
  8. killtracker/core/helpers.py +13 -0
  9. killtracker/core/{discord_messages.py → trackers.py} +74 -59
  10. killtracker/core/workers.py +46 -0
  11. killtracker/core/{killmails.py → zkb.py} +97 -72
  12. killtracker/forms.py +1 -1
  13. killtracker/managers.py +3 -3
  14. killtracker/models/trackers.py +7 -10
  15. killtracker/models/webhooks.py +60 -128
  16. killtracker/providers.py +1 -1
  17. killtracker/signals.py +31 -0
  18. killtracker/tasks.py +141 -92
  19. killtracker/tests/core/test_discord.py +184 -0
  20. killtracker/tests/core/test_helpers.py +23 -0
  21. killtracker/tests/core/{test_discord_messages_1.py → test_tracker_1.py} +28 -8
  22. killtracker/tests/core/{test_discord_messages_2.py → test_tracker_2.py} +11 -11
  23. killtracker/tests/core/test_workers.py +49 -0
  24. killtracker/tests/core/{test_killmails.py → test_zkb.py} +109 -52
  25. killtracker/tests/models/test_killmails.py +0 -2
  26. killtracker/tests/models/test_trackers_1.py +24 -24
  27. killtracker/tests/models/test_trackers_2.py +6 -5
  28. killtracker/tests/models/test_webhooks.py +63 -0
  29. killtracker/tests/test_integration.py +25 -12
  30. killtracker/tests/test_tasks.py +161 -92
  31. killtracker/tests/test_utils.py +39 -0
  32. killtracker/tests/testdata/factories.py +1 -1
  33. killtracker/tests/testdata/helpers.py +1 -1
  34. killtracker/tests/utils.py +44 -0
  35. killtracker/exceptions.py +0 -32
  36. killtracker/tests/models/test_webhook.py +0 -150
  37. killtracker/tests/test_exceptions.py +0 -12
  38. {aa_killtracker-0.17.0.dist-info → aa_killtracker-1.0.0.dist-info}/WHEEL +0 -0
  39. {aa_killtracker-0.17.0.dist-info → aa_killtracker-1.0.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,9 +1,12 @@
1
- """Create discord messages from killmails."""
1
+ """Generate Discord messages from tracked killmails."""
2
2
 
3
- from typing import NamedTuple, Optional
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import TYPE_CHECKING, Optional
4
7
 
5
8
  import dhooks_lite
6
- from requests.exceptions import HTTPError
9
+ import requests
7
10
 
8
11
  from eveuniverse.helpers import EveEntityNameResolver
9
12
  from eveuniverse.models import EveEntity, EveSolarSystem
@@ -16,33 +19,54 @@ from app_utils.urls import static_file_absolute_url
16
19
  from app_utils.views import humanize_value
17
20
 
18
21
  from killtracker import __title__
19
- from killtracker.models import Tracker
20
-
21
- from .killmails import ZKB_KILLMAIL_BASEURL, Killmail, TrackerInfo
22
+ from killtracker.core.discord import DiscordMessage
23
+ from killtracker.core.zkb import ZKB_KILLMAIL_BASEURL, Killmail, TrackerInfo
22
24
 
23
- ICON_SIZE = 128
25
+ if TYPE_CHECKING:
26
+ from killtracker.models import Tracker
24
27
 
28
+ _ICON_SIZE = 128
25
29
 
26
30
  logger = LoggerAddTag(get_extension_logger(__name__), __title__)
27
31
 
28
32
 
29
- class MainOrgInfo(NamedTuple):
33
+ def create_discord_message_from_killmail(
34
+ tracker: Tracker, killmail: Killmail, intro_text: Optional[str] = None
35
+ ) -> "DiscordMessage":
36
+ """Creates a Discord message from a Killmail and returns it."""
37
+ m = DiscordMessage(
38
+ killmail_id=killmail.id,
39
+ content=_create_content(tracker, intro_text),
40
+ embeds=[_create_embed(tracker, killmail)],
41
+ )
42
+ return m
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class _MainOrgInfo:
30
47
  """Infos about a main organization."""
31
48
 
32
- text: str = ""
49
+ icon_url: str = ""
33
50
  name: str = ""
34
- icon_url: str = eveimageserver.alliance_logo_url(1, size=ICON_SIZE)
35
51
  show_as_fleet_kill: bool = False
52
+ text: str = ""
53
+
54
+ def __post_init__(self):
55
+ if self.icon_url == "":
56
+ icon_url = eveimageserver.alliance_logo_url(1, size=_ICON_SIZE)
57
+ object.__setattr__(self, "icon_url", icon_url)
36
58
 
37
59
 
38
- class FinalAttackerInfo(NamedTuple):
60
+ @dataclass(frozen=True)
61
+ class _FinalAttackerInfo:
39
62
  """Infos about the final attacker on a killmail."""
40
63
 
41
64
  name: str = ""
42
65
  ship_type: str = ""
43
66
 
44
67
 
45
- class VictimInfo(NamedTuple):
68
+ @dataclass(frozen=True)
69
+ class _VictimInfo:
46
70
  """Infos about the victim of a killmail."""
47
71
 
48
72
  name: str
@@ -53,9 +77,11 @@ class VictimInfo(NamedTuple):
53
77
  ship_type_icon_url: str
54
78
 
55
79
 
56
- def create_content(tracker: Tracker, intro_text: Optional[str] = None) -> str:
80
+ def _create_content(tracker: Tracker, intro_text: Optional[str] = None) -> str:
57
81
  """Create content for Discord message for a killmail."""
58
82
 
83
+ from killtracker.models import Tracker
84
+
59
85
  intro_parts = []
60
86
 
61
87
  if tracker.ping_type == Tracker.ChannelPingType.EVERYBODY:
@@ -69,7 +95,7 @@ def create_content(tracker: Tracker, intro_text: Optional[str] = None) -> str:
69
95
  for group in tracker.ping_groups.all():
70
96
  try:
71
97
  role = DiscordUser.objects.group_to_role(group) # type: ignore
72
- except HTTPError:
98
+ except requests.exceptions.HTTPError:
73
99
  logger.warning(
74
100
  "Failed to get Discord roles. Can not ping groups.",
75
101
  exc_info=True,
@@ -102,7 +128,7 @@ def _import_discord_user():
102
128
  return DiscordUser
103
129
 
104
130
 
105
- def create_embed(tracker: Tracker, killmail: Killmail) -> dhooks_lite.Embed:
131
+ def _create_embed(tracker: Tracker, killmail: Killmail) -> dhooks_lite.Embed:
106
132
  """Create Discord embed for a killmail."""
107
133
 
108
134
  resolver: EveEntityNameResolver = EveEntity.objects.bulk_resolve_names( # type: ignore
@@ -111,7 +137,7 @@ def create_embed(tracker: Tracker, killmail: Killmail) -> dhooks_lite.Embed:
111
137
 
112
138
  # self info
113
139
  distance_text = ""
114
- main_org = MainOrgInfo()
140
+ main_org = _MainOrgInfo()
115
141
  main_ship_group_text = ""
116
142
  tracked_ship_types_text = ""
117
143
 
@@ -137,10 +163,24 @@ def create_embed(tracker: Tracker, killmail: Killmail) -> dhooks_lite.Embed:
137
163
  title = _calc_title(killmail, resolver, main_org, victim)
138
164
  thumbnail_url = _calc_thumbnail_url(victim, main_org)
139
165
 
140
- return _create_embed(killmail, tracker, victim, description, title, thumbnail_url)
166
+ author = _calc_author(victim)
167
+ zkb_icon_url = static_file_absolute_url("killtracker/zkb_icon.png")
168
+ embed_color = int(tracker.color[1:], 16) if tracker and tracker.color else None
169
+
170
+ embed = dhooks_lite.Embed(
171
+ author=author,
172
+ description=description,
173
+ title=title,
174
+ url=f"{ZKB_KILLMAIL_BASEURL}{killmail.id}/",
175
+ thumbnail=dhooks_lite.Thumbnail(url=thumbnail_url),
176
+ footer=dhooks_lite.Footer(text="zKillboard", icon_url=zkb_icon_url),
177
+ timestamp=killmail.time,
178
+ color=embed_color,
179
+ )
180
+ return embed
141
181
 
142
182
 
143
- def _calc_author(victim: VictimInfo):
183
+ def _calc_author(victim: _VictimInfo):
144
184
  # TODO This is a workaround for Embed.Author.name. Address in dhooks_lite
145
185
  return (
146
186
  dhooks_lite.Author(
@@ -158,10 +198,10 @@ def _calc_description(
158
198
  killmail: Killmail,
159
199
  resolver: EveEntityNameResolver,
160
200
  distance_text: str,
161
- main_org: MainOrgInfo,
201
+ main_org: _MainOrgInfo,
162
202
  main_ship_group_text: str,
163
203
  tracked_ship_types_text: str,
164
- victim: VictimInfo,
204
+ victim: _VictimInfo,
165
205
  ):
166
206
  solar_system_text = _calc_solar_system(tracker, killmail)
167
207
  total_value = (
@@ -186,18 +226,18 @@ def _calc_description(
186
226
 
187
227
  def _calc_victim(
188
228
  tracker: Tracker, killmail: Killmail, resolver: EveEntityNameResolver
189
- ) -> VictimInfo:
229
+ ) -> _VictimInfo:
190
230
  if killmail.victim.alliance_id:
191
231
  victim_organization = resolver.to_name(killmail.victim.alliance_id)
192
232
  victim_org_url = zkillboard.alliance_url(killmail.victim.alliance_id)
193
233
  victim_org_icon_url = eveimageserver.alliance_logo_url(
194
- killmail.victim.alliance_id, size=ICON_SIZE
234
+ killmail.victim.alliance_id, size=_ICON_SIZE
195
235
  )
196
236
  elif killmail.victim.corporation_id:
197
237
  victim_organization = resolver.to_name(killmail.victim.corporation_id)
198
238
  victim_org_url = zkillboard.corporation_url(killmail.victim.corporation_id)
199
239
  victim_org_icon_url = eveimageserver.corporation_logo_url(
200
- killmail.victim.corporation_id, size=ICON_SIZE
240
+ killmail.victim.corporation_id, size=_ICON_SIZE
201
241
  )
202
242
  else:
203
243
  victim_organization = ""
@@ -229,12 +269,12 @@ def _calc_victim(
229
269
  ship_type = resolver.to_name(ship_type_id) if ship_type_id else ""
230
270
 
231
271
  ship_type_icon_url = (
232
- eveimageserver.type_icon_url(ship_type_id, size=ICON_SIZE)
272
+ eveimageserver.type_icon_url(ship_type_id, size=_ICON_SIZE)
233
273
  if ship_type_id
234
274
  else ""
235
275
  )
236
276
 
237
- return VictimInfo(
277
+ return _VictimInfo(
238
278
  organization=victim_organization,
239
279
  org_url=victim_org_url,
240
280
  org_icon_url=victim_org_icon_url,
@@ -246,7 +286,7 @@ def _calc_victim(
246
286
 
247
287
  def _calc_final_attacker(
248
288
  tracker: Tracker, killmail: Killmail, resolver: EveEntityNameResolver
249
- ) -> FinalAttackerInfo:
289
+ ) -> _FinalAttackerInfo:
250
290
  for attacker in killmail.attackers:
251
291
  if attacker.is_final_blow:
252
292
  final_attacker = attacker
@@ -255,7 +295,7 @@ def _calc_final_attacker(
255
295
  final_attacker = None
256
296
 
257
297
  if not final_attacker:
258
- return FinalAttackerInfo()
298
+ return _FinalAttackerInfo()
259
299
 
260
300
  if final_attacker.corporation_id:
261
301
  final_attacker_corporation_zkb_link = _corporation_zkb_link(
@@ -286,7 +326,7 @@ def _calc_final_attacker(
286
326
 
287
327
  ship_type = resolver.to_name(ship_type_id) if ship_type_id else ""
288
328
 
289
- return FinalAttackerInfo(name=final_attacker_str, ship_type=ship_type)
329
+ return _FinalAttackerInfo(name=final_attacker_str, ship_type=ship_type)
290
330
 
291
331
 
292
332
  def _calc_solar_system(tracker: Tracker, killmail: Killmail):
@@ -338,12 +378,12 @@ def _calc_main_group(
338
378
  if main_org_entity.is_corporation:
339
379
  main_org_link = _corporation_zkb_link(tracker, main_org_entity.id, resolver)
340
380
  main_org_icon_url = eveimageserver.corporation_logo_url(
341
- main_org_entity.id, size=ICON_SIZE
381
+ main_org_entity.id, size=_ICON_SIZE
342
382
  )
343
383
  else:
344
384
  main_org_link = _alliance_zkb_link(tracker, main_org_entity.id, resolver)
345
385
  main_org_icon_url = eveimageserver.alliance_logo_url(
346
- main_org_entity.id, size=ICON_SIZE
386
+ main_org_entity.id, size=_ICON_SIZE
347
387
  )
348
388
  main_org_text = f" | Main group: {main_org_link} ({main_org_entity.count})"
349
389
  show_as_fleet_kill = tracker.identify_fleets
@@ -351,7 +391,7 @@ def _calc_main_group(
351
391
  show_as_fleet_kill = False
352
392
  main_org_text = main_org_name = main_org_icon_url = ""
353
393
 
354
- return MainOrgInfo(
394
+ return _MainOrgInfo(
355
395
  text=main_org_text,
356
396
  name=main_org_name,
357
397
  icon_url=main_org_icon_url,
@@ -380,7 +420,7 @@ def _calc_tracked_ship_types(
380
420
  return f"\nTracked ship types involved: **{ship_types_text}**"
381
421
 
382
422
 
383
- def _calc_thumbnail_url(victim: VictimInfo, main_org: MainOrgInfo):
423
+ def _calc_thumbnail_url(victim: _VictimInfo, main_org: _MainOrgInfo):
384
424
  if main_org.show_as_fleet_kill:
385
425
  return main_org.icon_url
386
426
 
@@ -390,8 +430,8 @@ def _calc_thumbnail_url(victim: VictimInfo, main_org: MainOrgInfo):
390
430
  def _calc_title(
391
431
  killmail: Killmail,
392
432
  resolver: EveEntityNameResolver,
393
- main_org: MainOrgInfo,
394
- victim: VictimInfo,
433
+ main_org: _MainOrgInfo,
434
+ victim: _VictimInfo,
395
435
  ):
396
436
  solar_system_name = (
397
437
  resolver.to_name(killmail.solar_system_id) if killmail.solar_system_id else ""
@@ -403,31 +443,6 @@ def _calc_title(
403
443
  return f"{solar_system_name} | {victim.ship_type} | Killmail"
404
444
 
405
445
 
406
- def _create_embed(
407
- killmail: Killmail,
408
- tracker: Tracker,
409
- victim: VictimInfo,
410
- description: str,
411
- title: str,
412
- thumbnail_url: str,
413
- ):
414
- author = _calc_author(victim)
415
- zkb_icon_url = static_file_absolute_url("killtracker/zkb_icon.png")
416
- embed_color = int(tracker.color[1:], 16) if tracker and tracker.color else None
417
-
418
- embed = dhooks_lite.Embed(
419
- author=author,
420
- description=description,
421
- title=title,
422
- url=f"{ZKB_KILLMAIL_BASEURL}{killmail.id}/",
423
- thumbnail=dhooks_lite.Thumbnail(url=thumbnail_url),
424
- footer=dhooks_lite.Footer(text="zKillboard", icon_url=zkb_icon_url),
425
- timestamp=killmail.time,
426
- color=embed_color,
427
- )
428
- return embed
429
-
430
-
431
446
  def _character_zkb_link(
432
447
  tracker: Tracker, entity_id: int, resolver: EveEntityNameResolver
433
448
  ) -> str:
@@ -0,0 +1,46 @@
1
+ """Allows tasks to find out whether their worker is currently shutting down.
2
+
3
+ This enables long running tasks to abort early,
4
+ which helps to speed up a warm worker shutdown.
5
+ """
6
+
7
+ from celery import Task
8
+
9
+ from django.core.cache import cache
10
+
11
+ from allianceauth.services.hooks import get_extension_logger
12
+ from app_utils.logging import LoggerAddTag
13
+
14
+ from killtracker import __title__
15
+
16
+ _TIMEOUT_SECONDS = 120
17
+
18
+ logger = LoggerAddTag(get_extension_logger(__name__), __title__)
19
+
20
+
21
+ def state_reset(hostname: str) -> None:
22
+ """Resets the shutting down state for a worker."""
23
+ cache.delete(_make_key(hostname))
24
+
25
+
26
+ def state_set(hostname: str) -> None:
27
+ """Sets a worker into the shutting down state."""
28
+ cache.set(_make_key(hostname), "shutting down", timeout=_TIMEOUT_SECONDS)
29
+
30
+
31
+ def is_shutting_down(task: Task) -> bool:
32
+ """Reports whether the worker of a celery task is currently shutting down."""
33
+ try:
34
+ hostname = str(task.request.hostname)
35
+ except (AttributeError, TypeError, ValueError):
36
+ logger.warning("Failed to retrieve hostname: %s", task)
37
+ return False
38
+
39
+ if cache.get(_make_key(hostname)) is None:
40
+ return False
41
+
42
+ return True
43
+
44
+
45
+ def _make_key(hostname: str) -> str:
46
+ return f"killtracker-worker-shutting-down-{hostname}"
@@ -1,54 +1,69 @@
1
- """Fetching killmails from ZKB."""
1
+ """Fetch killmails from zKillboard."""
2
2
 
3
3
  # pylint: disable = redefined-builtin
4
4
 
5
+ import datetime as dt
5
6
  import json
6
7
  from copy import deepcopy
7
8
  from dataclasses import asdict, dataclass
8
- from datetime import datetime
9
9
  from http import HTTPStatus
10
+ from time import sleep
10
11
  from typing import List, Optional, Set
11
12
  from urllib.parse import quote_plus
12
13
 
13
14
  import requests
14
15
  from dacite import DaciteError, from_dict
15
- from redis.exceptions import LockError
16
16
  from simplejson.errors import JSONDecodeError
17
17
 
18
- from django.conf import settings
19
18
  from django.core.cache import cache
20
19
  from django.core.exceptions import ImproperlyConfigured
21
20
  from django.utils.dateparse import parse_datetime
21
+ from django.utils.timezone import now
22
22
  from eveuniverse.models import EveType
23
23
 
24
24
  from allianceauth.services.hooks import get_extension_logger
25
- from app_utils.allianceauth import get_redis_client
26
25
  from app_utils.json import JSONDateTimeDecoder, JSONDateTimeEncoder
27
26
  from app_utils.logging import LoggerAddTag
28
27
 
29
28
  from killtracker import USER_AGENT_TEXT, __title__
30
29
  from killtracker.app_settings import (
31
30
  KILLTRACKER_QUEUE_ID,
32
- KILLTRACKER_REDISQ_LOCK_TIMEOUT,
33
31
  KILLTRACKER_REDISQ_TTW,
34
32
  KILLTRACKER_STORAGE_KILLMAILS_LIFETIME,
33
+ KILLTRACKER_ZKB_REQUEST_DELAY,
35
34
  )
36
- from killtracker.exceptions import KillmailDoesNotExist
35
+ from killtracker.core.helpers import datetime_or_none
37
36
  from killtracker.providers import esi
38
37
 
39
- logger = LoggerAddTag(get_extension_logger(__name__), __title__)
40
-
41
- ZKB_REDISQ_URL = "https://zkillredisq.stream/listen.php"
42
- ZKB_API_URL = "https://zkillboard.com/api/"
43
38
  ZKB_KILLMAIL_BASEURL = "https://zkillboard.com/kill/"
44
- REQUESTS_TIMEOUT = (5, 30)
45
39
 
46
- MAIN_MINIMUM_COUNT = 2
47
- MAIN_MINIMUM_SHARE = 0.25
40
+ _KEY_RETRY_AT = "killtracker-zkb-retry-at"
41
+ _KEY_LAST_REQUEST = "killtracker-zkb-last-request"
42
+ _MAIN_MINIMUM_COUNT = 2
43
+ _MAIN_MINIMUM_SHARE = 0.25
44
+ _REQUESTS_TIMEOUT = (5, 30)
45
+ _ZKB_429_DEFAULT_TIMEOUT = 10
46
+ _ZKB_API_URL = "https://zkillboard.com/api/"
47
+ _ZKB_REDISQ_URL = "https://zkillredisq.stream/listen.php"
48
+
49
+ logger = LoggerAddTag(get_extension_logger(__name__), __title__)
50
+
48
51
 
49
52
  # TODO: Factor out logic for accessing the API to another module
50
53
 
51
54
 
55
+ class ZKBTooManyRequestsError(Exception):
56
+ """ZKB RedisQ API has returned 429 Too Many Requests HTTP status code."""
57
+
58
+ def __init__(self, retry_at: dt.datetime, is_original: bool = True):
59
+ self.retry_at = retry_at
60
+ self.is_original = is_original
61
+
62
+
63
+ class KillmailDoesNotExist(Exception):
64
+ """Killmail does not exist in storage."""
65
+
66
+
52
67
  @dataclass
53
68
  class _KillmailBase:
54
69
  """Base class for all Killmail."""
@@ -160,7 +175,7 @@ class Killmail(_KillmailBase):
160
175
  _STORAGE_BASE_KEY = "killtracker_storage_killmail_"
161
176
 
162
177
  id: int
163
- time: datetime
178
+ time: dt.datetime
164
179
  victim: KillmailVictim
165
180
  attackers: List[KillmailAttacker]
166
181
  position: KillmailPosition
@@ -256,8 +271,8 @@ class Killmail(_KillmailBase):
256
271
  jumps: Optional[int] = None,
257
272
  distance: Optional[float] = None,
258
273
  matching_ship_type_ids: Optional[List[int]] = None,
259
- minimum_count: int = MAIN_MINIMUM_COUNT,
260
- minimum_share: float = MAIN_MINIMUM_SHARE,
274
+ minimum_count: int = _MAIN_MINIMUM_COUNT,
275
+ minimum_share: float = _MAIN_MINIMUM_SHARE,
261
276
  ) -> "Killmail":
262
277
  """Clone this killmail and add tracker info."""
263
278
  main_ship_group = self._calc_main_attacker_ship_group(
@@ -363,7 +378,10 @@ class Killmail(_KillmailBase):
363
378
 
364
379
  @classmethod
365
380
  def get(cls, id: int) -> "Killmail":
366
- """Fetch a killmail from temporary storage."""
381
+ """Fetch a killmail from temporary storage.
382
+
383
+ Raises KillmailDoesNotExist if killmail does not exit.
384
+ """
367
385
  data = cache.get(key=cls._storage_key(id))
368
386
  if not data:
369
387
  raise KillmailDoesNotExist(
@@ -391,9 +409,14 @@ class Killmail(_KillmailBase):
391
409
 
392
410
  @classmethod
393
411
  def create_from_zkb_redisq(cls) -> Optional["Killmail"]:
394
- """Fetches and returns a killmail from ZKB.
412
+ """Fetches and returns a killmail from ZKB REDISQ API.
413
+
414
+ Will automatically wait for a free rate limit slot if needed.
415
+ Will re-raise TooManyRequests if a recent 429 timeout is not yet expired.
395
416
 
396
- Returns None if no killmail is received.
417
+ This method is not thread safe.
418
+
419
+ Returns None if no killmail was received.
397
420
  """
398
421
  if not KILLTRACKER_QUEUE_ID:
399
422
  raise ImproperlyConfigured(
@@ -403,51 +426,70 @@ class Killmail(_KillmailBase):
403
426
  if "," in KILLTRACKER_QUEUE_ID:
404
427
  raise ImproperlyConfigured("A queue ID must not contains commas.")
405
428
 
406
- redis = get_redis_client()
407
- params = {
408
- "queueID": quote_plus(KILLTRACKER_QUEUE_ID),
409
- "ttw": KILLTRACKER_REDISQ_TTW,
410
- }
411
- try:
412
- logger.info("Trying to fetch killmail from ZKB RedisQ...")
413
- with redis.lock(
414
- cls.lock_key(), blocking_timeout=KILLTRACKER_REDISQ_LOCK_TIMEOUT
415
- ):
416
- response = requests.get(
417
- ZKB_REDISQ_URL,
418
- params=params,
419
- timeout=REQUESTS_TIMEOUT,
420
- headers={"User-Agent": USER_AGENT_TEXT},
421
- )
422
- except LockError:
429
+ retry_at = datetime_or_none(cache.get(_KEY_RETRY_AT))
430
+ if retry_at is not None and retry_at > now():
431
+ raise ZKBTooManyRequestsError(retry_at=retry_at, is_original=False)
432
+
433
+ last_request = datetime_or_none(cache.get(_KEY_LAST_REQUEST))
434
+ if last_request is not None:
435
+ next_slot = last_request + dt.timedelta(
436
+ milliseconds=KILLTRACKER_ZKB_REQUEST_DELAY
437
+ )
438
+ seconds = (next_slot - now()).total_seconds()
439
+ if seconds > 0:
440
+ logger.debug("ZKB API: Waiting %f seconds for next free slot", seconds)
441
+ sleep(seconds)
442
+
443
+ response = requests.get(
444
+ _ZKB_REDISQ_URL,
445
+ params={
446
+ "queueID": quote_plus(KILLTRACKER_QUEUE_ID),
447
+ "ttw": KILLTRACKER_REDISQ_TTW,
448
+ },
449
+ timeout=_REQUESTS_TIMEOUT,
450
+ headers={"User-Agent": USER_AGENT_TEXT},
451
+ )
452
+ cache.set(_KEY_LAST_REQUEST, now(), timeout=KILLTRACKER_ZKB_REQUEST_DELAY + 30)
453
+ logger.debug(
454
+ "Response from ZKB API: %d %s %s",
455
+ response.status_code,
456
+ response.headers,
457
+ response.text,
458
+ )
459
+
460
+ if not response.ok:
423
461
  logger.warning(
424
- "Failed to acquire lock for atomic access to RedisQ.",
425
- exc_info=settings.DEBUG, # provide details in DEBUG mode
462
+ "ZKB API returned error: %d %s", response.status_code, response.text
426
463
  )
427
- return None
464
+ if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
465
+ try:
466
+ retry_after = int(response.headers["Retry-After"])
467
+ except KeyError:
468
+ retry_after = _ZKB_429_DEFAULT_TIMEOUT
469
+ retry_at = now() + dt.timedelta(seconds=retry_after)
470
+ cache.set(_KEY_RETRY_AT, retry_at, timeout=retry_after + 60)
471
+ raise ZKBTooManyRequestsError(retry_at=retry_at, is_original=True)
428
472
 
429
- if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
430
- logger.error("429 Client Error: Too many requests: %s", response.text)
431
473
  return None
432
474
 
433
- response.raise_for_status()
434
-
435
475
  try:
436
476
  data = response.json()
437
477
  except JSONDecodeError:
438
- logger.error("Error from ZKB API:\n%s", response.text)
478
+ logger.error("Error parsing ZKB API response:\n%s", response.text)
439
479
  return None
440
480
 
441
- if data:
442
- logger.debug("data:\n%s", data)
481
+ if not data or "package" not in data or not data["package"]:
482
+ logger.info("ZKB did not return a killmail")
483
+ return None
443
484
 
444
- if data and "package" in data and data["package"]:
445
- logger.info("Received a killmail from ZKB RedisQ")
446
- package_data = data["package"]
447
- return cls._create_from_dict(package_data)
485
+ package_data = data["package"]
486
+ km = cls._create_from_dict(package_data)
487
+ if km is not None:
488
+ logger.info("ZKB returned killmail %d", km.id)
489
+ else:
490
+ logger.info("Failed to parse killmail from ZKB")
448
491
 
449
- logger.debug("Did not received a killmail from ZKB RedisQ")
450
- return None
492
+ return km
451
493
 
452
494
  @classmethod
453
495
  def create_from_zkb_api(cls, killmail_id: int) -> Optional["Killmail"]:
@@ -464,9 +506,9 @@ class Killmail(_KillmailBase):
464
506
  "Trying to fetch killmail from ZKB API with killmail ID %d ...",
465
507
  killmail_id,
466
508
  )
467
- url = f"{ZKB_API_URL}killID/{killmail_id}/"
509
+ url = f"{_ZKB_API_URL}killID/{killmail_id}/"
468
510
  response = requests.get(
469
- url, timeout=REQUESTS_TIMEOUT, headers={"User-Agent": USER_AGENT_TEXT}
511
+ url, timeout=_REQUESTS_TIMEOUT, headers={"User-Agent": USER_AGENT_TEXT}
470
512
  )
471
513
  response.raise_for_status()
472
514
  zkb_data = response.json()
@@ -601,20 +643,3 @@ class Killmail(_KillmailBase):
601
643
  params[prop] = zkb_data[prop]
602
644
 
603
645
  return KillmailZkb(**params)
604
-
605
- @staticmethod
606
- def lock_key() -> str:
607
- """Key used for lock operation on Redis."""
608
- return f"{__title__.upper()}_REDISQ_LOCK"
609
-
610
- @classmethod
611
- def reset_lock_key(cls):
612
- """Delete lock key if it exists.
613
-
614
- It can happen that a lock key is not cleaned up
615
- and then prevents this class from ever acquiring a lock again.
616
- To prevent this we are deleting the lock key at system start.
617
- """
618
- redis = get_redis_client()
619
- if redis.delete(cls.lock_key()) > 0:
620
- logger.warning("A stuck lock key was cleared.")
killtracker/forms.py CHANGED
@@ -7,7 +7,7 @@ from django.core.exceptions import ValidationError
7
7
  from django.forms.widgets import TextInput
8
8
  from django.utils.translation import gettext_lazy as _
9
9
 
10
- from .models import Tracker
10
+ from killtracker.models import Tracker
11
11
 
12
12
 
13
13
  def field_nice_display(name: str) -> str:
killtracker/managers.py CHANGED
@@ -13,9 +13,9 @@ from allianceauth.services.hooks import get_extension_logger
13
13
  from app_utils.caching import ObjectCacheMixin
14
14
  from app_utils.logging import LoggerAddTag
15
15
 
16
- from . import __title__
17
- from .app_settings import KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS
18
- from .core.killmails import Killmail, _KillmailCharacter
16
+ from killtracker import __title__
17
+ from killtracker.app_settings import KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS
18
+ from killtracker.core.zkb import Killmail, _KillmailCharacter
19
19
 
20
20
  logger = LoggerAddTag(get_extension_logger(__name__), __title__)
21
21
 
@@ -28,10 +28,10 @@ from app_utils.logging import LoggerAddTag
28
28
  from killtracker import __title__
29
29
  from killtracker.app_settings import KILLTRACKER_KILLMAIL_MAX_AGE_FOR_TRACKER
30
30
  from killtracker.constants import EveCategoryId, EveGroupId
31
- from killtracker.core.killmails import Killmail
31
+ from killtracker.core.trackers import create_discord_message_from_killmail
32
+ from killtracker.core.zkb import Killmail
32
33
  from killtracker.managers import TrackerManager
33
-
34
- from .webhooks import Webhook
34
+ from killtracker.models.webhooks import Webhook
35
35
 
36
36
  logger = LoggerAddTag(get_extension_logger(__name__), __title__)
37
37
 
@@ -881,12 +881,9 @@ class Tracker(models.Model):
881
881
  def generate_killmail_message(
882
882
  self, killmail: Killmail, intro_text: Optional[str] = None
883
883
  ) -> int:
884
- """generate a message from given killmail and enqueue for later sending
884
+ """Generate a message from given killmail and enqueue for later sending.
885
885
 
886
- returns new queue size
886
+ Returns the new queue size.
887
887
  """
888
- from killtracker.core import discord_messages
889
-
890
- content = discord_messages.create_content(self, intro_text)
891
- embed = discord_messages.create_embed(self, killmail)
892
- return self.webhook.enqueue_message(content=content, embeds=[embed])
888
+ message = create_discord_message_from_killmail(self, killmail, intro_text)
889
+ return self.webhook.enqueue_message(message)