aa-killtracker 0.18.0a2__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/PKG-INFO +7 -8
  2. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/README.md +0 -1
  3. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/__init__.py +1 -1
  4. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/admin.py +1 -1
  5. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/app_settings.py +14 -7
  6. aa_killtracker-1.0.0/killtracker/core/discord.py +162 -0
  7. aa_killtracker-1.0.0/killtracker/core/helpers.py +13 -0
  8. aa_killtracker-0.18.0a2/killtracker/core/discord_messages.py → aa_killtracker-1.0.0/killtracker/core/trackers.py +16 -75
  9. aa_killtracker-0.18.0a2/killtracker/core/worker_shutdown.py → aa_killtracker-1.0.0/killtracker/core/workers.py +3 -4
  10. aa_killtracker-0.18.0a2/killtracker/core/killmails.py → aa_killtracker-1.0.0/killtracker/core/zkb.py +32 -40
  11. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/managers.py +1 -1
  12. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/models/trackers.py +4 -5
  13. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/models/webhooks.py +4 -53
  14. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/signals.py +4 -4
  15. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tasks.py +47 -49
  16. aa_killtracker-1.0.0/killtracker/tests/core/test_discord.py +184 -0
  17. aa_killtracker-1.0.0/killtracker/tests/core/test_helpers.py +23 -0
  18. aa_killtracker-0.18.0a2/killtracker/tests/core/test_discord_messages_1.py → aa_killtracker-1.0.0/killtracker/tests/core/test_tracker_1.py +12 -39
  19. aa_killtracker-0.18.0a2/killtracker/tests/core/test_discord_messages_2.py → aa_killtracker-1.0.0/killtracker/tests/core/test_tracker_2.py +3 -3
  20. aa_killtracker-1.0.0/killtracker/tests/core/test_workers.py +49 -0
  21. aa_killtracker-0.18.0a2/killtracker/tests/core/test_killmails.py → aa_killtracker-1.0.0/killtracker/tests/core/test_zkb.py +58 -46
  22. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/models/test_killmails.py +0 -2
  23. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/models/test_trackers_1.py +1 -1
  24. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/models/test_trackers_2.py +2 -2
  25. aa_killtracker-1.0.0/killtracker/tests/models/test_webhooks.py +63 -0
  26. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/test_integration.py +26 -13
  27. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/test_tasks.py +68 -58
  28. aa_killtracker-1.0.0/killtracker/tests/test_utils.py +39 -0
  29. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/factories.py +1 -1
  30. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/helpers.py +1 -1
  31. aa_killtracker-1.0.0/killtracker/tests/utils.py +44 -0
  32. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/pyproject.toml +6 -6
  33. aa_killtracker-0.18.0a2/killtracker/exceptions.py +0 -32
  34. aa_killtracker-0.18.0a2/killtracker/tests/core/test_worker_shutdown.py +0 -34
  35. aa_killtracker-0.18.0a2/killtracker/tests/models/test_webhook.py +0 -164
  36. aa_killtracker-0.18.0a2/killtracker/tests/test_exceptions.py +0 -12
  37. aa_killtracker-0.18.0a2/killtracker/tests/utils.py +0 -16
  38. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/LICENSE +0 -0
  39. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/apps.py +0 -0
  40. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/checks.py +0 -0
  41. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/constants.py +0 -0
  42. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/core/__init__.py +0 -0
  43. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/forms.py +0 -0
  44. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/management/commands/killtracker_load_eve.py +0 -0
  45. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0001_initial_new.py +0 -0
  46. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0001_squashed_all.py +0 -0
  47. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0002_fix_webhook_notes_field.py +0 -0
  48. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0002_tracker_require_attackers_weapon_groups_and_more.py +0 -0
  49. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0003_add_state_clauses.py +0 -0
  50. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0003_optimize_tracker_form.py +0 -0
  51. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0004_add_faction_clauses.py +0 -0
  52. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0004_django4_update.py +0 -0
  53. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0005_add_final_blow_clause_and_more.py +0 -0
  54. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0006_evetypeplus.py +0 -0
  55. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0007_restructure_killsmails.py +0 -0
  56. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0008_copy_data_to_new_structure.py +0 -0
  57. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/0009_remove_old_models.py +0 -0
  58. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/migrations/__init__.py +0 -0
  59. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/models/__init__.py +0 -0
  60. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/models/killmails.py +0 -0
  61. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/providers.py +0 -0
  62. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/static/killtracker/killtracker_logo.png +0 -0
  63. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/static/killtracker/zkb_icon.png +0 -0
  64. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/templates/admin/killtracker/tracker/killmail_test.html +0 -0
  65. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/__init__.py +0 -0
  66. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/core/__init__.py +0 -0
  67. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/models/__init__.py +0 -0
  68. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/test_admin.py +0 -0
  69. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/test_admin_2.py +0 -0
  70. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/__init__.py +0 -0
  71. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/create_eveuniverse.py +0 -0
  72. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/evealliances.json +0 -0
  73. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/evecorporations.json +0 -0
  74. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/eveentities.json +0 -0
  75. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/eveuniverse.json +0 -0
  76. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/killmails.json +0 -0
  77. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tests/testdata/load_eveuniverse.py +0 -0
  78. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tools/drop_tables_killtracker.sql +0 -0
  79. {aa_killtracker-0.18.0a2 → aa_killtracker-1.0.0}/killtracker/tools/generate_conditions_text.py +0 -0
@@ -1,13 +1,12 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: aa-killtracker
3
- Version: 0.18.0a2
3
+ Version: 1.0.0
4
4
  Summary: An app for running killmail trackers with Alliance Auth and Discord.
5
5
  Author-email: Erik Kalkoken <kalkoken87@gmail.com>
6
6
  Requires-Python: >=3.8
7
7
  Description-Content-Type: text/markdown
8
8
  Classifier: Environment :: Web Environment
9
9
  Classifier: Framework :: Django
10
- Classifier: Framework :: Django :: 4.0
11
10
  Classifier: Framework :: Django :: 4.2
12
11
  Classifier: Intended Audience :: End Users/Desktop
13
12
  Classifier: License :: OSI Approved :: MIT License
@@ -17,15 +16,16 @@ Classifier: Programming Language :: Python :: 3.8
17
16
  Classifier: Programming Language :: Python :: 3.9
18
17
  Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
20
  Classifier: Topic :: Internet :: WWW/HTTP
21
21
  Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
22
22
  License-File: LICENSE
23
- Requires-Dist: allianceauth-app-utils>=1.19.0
24
- Requires-Dist: allianceauth>=3
23
+ Requires-Dist: allianceauth-app-utils>=1.26
24
+ Requires-Dist: allianceauth>=4,<5
25
25
  Requires-Dist: dacite
26
- Requires-Dist: dhooks-lite>=1.0
27
- Requires-Dist: django-eveuniverse>=1.3
28
- Requires-Dist: redis-simple-mq>=0.5
26
+ Requires-Dist: dhooks-lite>=1.1
27
+ Requires-Dist: django-eveuniverse>=1.5
28
+ Requires-Dist: redis-simple-mq>=1.0
29
29
  Project-URL: Home, https://gitlab.com/ErikKalkoken/aa-killtracker
30
30
 
31
31
  # Killtracker
@@ -242,7 +242,6 @@ Note that all settings are optional and the app will use the documented default
242
242
  Name | Description | Default
243
243
  -- | -- | --
244
244
  `KILLTRACKER_KILLMAIL_MAX_AGE_FOR_TRACKER`| Ignore killmails that are older than the given number in minutes. Sometimes killmails appear belated on ZKB, this feature ensures they don't create new alerts | `60`
245
- `KILLTRACKER_MAX_KILLMAILS_PER_RUN`| Maximum number of killmails retrieved from ZKB by task run. This value should be set such that the task that fetches new killmails from ZKB every minute will reliable finish within one minute. To test this run a "Catch all" tracker and see how many killmails your system is capable of processing. Note that you can get that information from the worker's log file. It will look something like this: `Total killmails received from ZKB in 49 secs: 100` | `100`
246
245
  `KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS`| Killmails older than set number of days will be purged from the database. If you want to keep all killmails set this to 0. Note that this setting is only relevant if you have storing killmails enabled. | `30`
247
246
  `KILLTRACKER_QUEUE_ID`| Unique ID used to identify this server when fetching killmails from zKillboard. This setting is mandatory. | ``
248
247
  `KILLTRACKER_STORING_KILLMAILS_ENABLED`| If set to true Killtracker will automatically store all received killmails in the local database. This can be useful if you want to run analytics on killmails etc. However, please note that Killtracker itself currently does not use stored killmails in any way. | `False`
@@ -212,7 +212,6 @@ Note that all settings are optional and the app will use the documented default
212
212
  Name | Description | Default
213
213
  -- | -- | --
214
214
  `KILLTRACKER_KILLMAIL_MAX_AGE_FOR_TRACKER`| Ignore killmails that are older than the given number in minutes. Sometimes killmails appear belated on ZKB, this feature ensures they don't create new alerts | `60`
215
- `KILLTRACKER_MAX_KILLMAILS_PER_RUN`| Maximum number of killmails retrieved from ZKB by task run. This value should be set such that the task that fetches new killmails from ZKB every minute will reliable finish within one minute. To test this run a "Catch all" tracker and see how many killmails your system is capable of processing. Note that you can get that information from the worker's log file. It will look something like this: `Total killmails received from ZKB in 49 secs: 100` | `100`
216
215
  `KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS`| Killmails older than set number of days will be purged from the database. If you want to keep all killmails set this to 0. Note that this setting is only relevant if you have storing killmails enabled. | `30`
217
216
  `KILLTRACKER_QUEUE_ID`| Unique ID used to identify this server when fetching killmails from zKillboard. This setting is mandatory. | ``
218
217
  `KILLTRACKER_STORING_KILLMAILS_ENABLED`| If set to true Killtracker will automatically store all received killmails in the local database. This can be useful if you want to run analytics on killmails etc. However, please note that Killtracker itself currently does not use stored killmails in any way. | `False`
@@ -3,7 +3,7 @@
3
3
  # pylint: disable = invalid-name
4
4
  default_app_config = "killtracker.apps.KillmailsConfig"
5
5
 
6
- __version__ = "0.18.0a2"
6
+ __version__ = "1.0.0"
7
7
  __title__ = "Killtracker"
8
8
 
9
9
  APP_NAME = "aa-killtracker"
@@ -12,7 +12,7 @@ from django.utils.safestring import mark_safe
12
12
  from allianceauth import NAME as site_header
13
13
 
14
14
  from killtracker import tasks
15
- from killtracker.core.killmails import Killmail
15
+ from killtracker.core.zkb import Killmail
16
16
  from killtracker.forms import (
17
17
  TrackerAdminForm,
18
18
  TrackerAdminKillmailIdForm,
@@ -13,13 +13,8 @@ sometimes killmails appear belated on ZKB,
13
13
  this feature ensures they don't create new alerts.
14
14
  """
15
15
 
16
- KILLTRACKER_MAX_KILLMAILS_PER_RUN = clean_setting(
17
- "KILLTRACKER_MAX_KILLMAILS_PER_RUN", 500
18
- )
19
- """Maximum number of killmails retrieved from ZKB by task run."""
20
-
21
16
  KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS = clean_setting(
22
- "KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS", 30
17
+ "KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS", default_value=30, min_value=0
23
18
  )
24
19
  """Killmails older than set number of days will be purged from the database.
25
20
  If you want to keep all killmails set this to 0.
@@ -103,9 +98,21 @@ KILLTRACKER_STORAGE_KILLMAILS_LIFETIME = clean_setting(
103
98
  )
104
99
  """Max lifetime of killmails in temporary storage in seconds."""
105
100
 
106
- KILLTRACKER_ZKB_REQUEST_DELAY = clean_setting("KILLTRACKER_ZKB_REQUEST_DELAY", 500)
101
+ KILLTRACKER_ZKB_REQUEST_DELAY = clean_setting(
102
+ "KILLTRACKER_ZKB_REQUEST_DELAY", default_value=500, min_value=500
103
+ )
107
104
  """Delay between subsequent calls to ZKB API in milliseconds.
108
105
 
109
106
  This delay ensures the app does not breach the CloudFlare rate limit of currently
110
107
  two (2) requests per second per IP address.
111
108
  """
109
+
110
+ KILLTRACKER_MAX_KILLMAILS_PER_RUN = clean_setting(
111
+ "KILLTRACKER_MAX_KILLMAILS_PER_RUN", default_value=500, min_value=1
112
+ )
113
+ """Maximum number of killmails retrieved from ZKB by task run."""
114
+
115
+ KILLTRACKER_MAX_MESSAGES_SENT_PER_RUN = clean_setting(
116
+ "KILLTRACKER_MAX_MESSAGES_SENT_PER_RUN", default_value=10, min_value=1
117
+ )
118
+ """Maximum number of messages processed per task run."""
@@ -0,0 +1,162 @@
1
+ """Send messages to Discord webhooks."""
2
+
3
+ import datetime as dt
4
+ import json
5
+ from copy import copy
6
+ from dataclasses import dataclass
7
+ from http import HTTPStatus
8
+ from time import sleep
9
+ from typing import List, Optional
10
+
11
+ import dhooks_lite
12
+
13
+ from django.core.cache import cache
14
+ from django.utils.timezone import now
15
+
16
+ from allianceauth.services.hooks import get_extension_logger
17
+ from app_utils.json import JSONDateTimeDecoder, JSONDateTimeEncoder
18
+ from app_utils.logging import LoggerAddTag
19
+
20
+ from killtracker import APP_NAME, HOMEPAGE_URL, __title__, __version__
21
+ from killtracker.app_settings import KILLTRACKER_DISCORD_SEND_DELAY
22
+ from killtracker.core.helpers import datetime_or_none
23
+
24
+ _DEFAULT_429_TIMEOUT = 600
25
+
26
+ logger = LoggerAddTag(get_extension_logger(__name__), __title__)
27
+
28
+
29
+ class HTTPError(Exception):
30
+ """A HTTP error."""
31
+
32
+ def __init__(self, status_code: int):
33
+ self.status_code = status_code
34
+
35
+
36
+ class WebhookRateLimitExhausted(Exception):
37
+ """The rate limit of a webhook has been exhausted."""
38
+
39
+ def __init__(self, retry_at: dt.datetime, is_original: bool = True):
40
+ self.retry_at = retry_at
41
+ self.is_original = is_original
42
+
43
+
44
+ @dataclass
45
+ class DiscordMessage:
46
+ """A Discord message created from a Killmail."""
47
+
48
+ avatar_url: Optional[str] = None
49
+ content: Optional[str] = None
50
+ embeds: Optional[List[dhooks_lite.Embed]] = None
51
+ killmail_id: int = 0 # Killmail ID this message from created from
52
+ username: Optional[str] = None
53
+
54
+ def __post_init__(self):
55
+ if not self.content and not self.embeds:
56
+ raise ValueError("Message must have content or embeds to be valid")
57
+
58
+ def to_json(self) -> str:
59
+ """Converts a Discord message into a JSON object and returns it."""
60
+
61
+ if self.embeds:
62
+ embeds_list = [obj.asdict() for obj in self.embeds]
63
+ else:
64
+ embeds_list = None
65
+
66
+ message = {}
67
+ if self.killmail_id:
68
+ message["killmail_id"] = self.killmail_id
69
+ if self.content:
70
+ message["content"] = self.content
71
+ if embeds_list:
72
+ message["embeds"] = embeds_list
73
+ if self.username:
74
+ message["username"] = self.username
75
+ if self.avatar_url:
76
+ message["avatar_url"] = self.avatar_url
77
+
78
+ return json.dumps(message, cls=JSONDateTimeEncoder)
79
+
80
+ @classmethod
81
+ def from_json(cls, s: str) -> "DiscordMessage":
82
+ """Creates a DiscordMessage object from an JSON object and returns it."""
83
+ message1: dict = json.loads(s, cls=JSONDateTimeDecoder)
84
+ message2 = copy(message1)
85
+ if message1.get("embeds"):
86
+ message2["embeds"] = [
87
+ dhooks_lite.Embed.from_dict(embed_dict)
88
+ for embed_dict in message1.get("embeds")
89
+ ]
90
+ else:
91
+ message2["embeds"] = None
92
+ return cls(**message2)
93
+
94
+
95
+ def send_message_to_webhook(name: str, url: str, message: DiscordMessage) -> int:
96
+ """Send a message to a Discord webhook and returns the ID of new message."""
97
+
98
+ key_retry_at = _make_key_retry_at(url)
99
+ retry_at = datetime_or_none(cache.get(key_retry_at))
100
+ if retry_at is not None and retry_at > now():
101
+ raise WebhookRateLimitExhausted(retry_at=retry_at, is_original=False)
102
+
103
+ key_last_request = _make_key_last_request(url)
104
+ last_request = datetime_or_none(cache.get(key_last_request))
105
+ if last_request is not None:
106
+ next_slot = last_request + dt.timedelta(seconds=KILLTRACKER_DISCORD_SEND_DELAY)
107
+ seconds = (next_slot - now()).total_seconds()
108
+ if seconds > 0:
109
+ logger.debug(
110
+ "%s: Waiting %f seconds for next free slot for webhook", name, seconds
111
+ )
112
+ sleep(seconds)
113
+
114
+ hook = dhooks_lite.Webhook(
115
+ url=url,
116
+ user_agent=dhooks_lite.UserAgent(
117
+ name=APP_NAME, url=HOMEPAGE_URL, version=__version__
118
+ ),
119
+ )
120
+ response = hook.execute(
121
+ content=message.content,
122
+ embeds=message.embeds,
123
+ username=message.username,
124
+ avatar_url=message.avatar_url,
125
+ wait_for_response=True,
126
+ max_retries=0, # we will handle retries ourselves
127
+ )
128
+ cache.set(key_last_request, now(), timeout=KILLTRACKER_DISCORD_SEND_DELAY + 30)
129
+ logger.debug(
130
+ "%s: Response from Discord for creating message from killmail %d: %s %s %s",
131
+ name,
132
+ message.killmail_id,
133
+ response.status_code,
134
+ response.headers,
135
+ response.content,
136
+ )
137
+ if not response.status_ok:
138
+ if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
139
+ try:
140
+ retry_after = int(response.headers["Retry-After"])
141
+ except KeyError:
142
+ retry_after = _DEFAULT_429_TIMEOUT
143
+ retry_at = now() + dt.timedelta(seconds=retry_after)
144
+ cache.set(key_retry_at, retry_at, timeout=retry_after + 60)
145
+ raise WebhookRateLimitExhausted(retry_at=retry_at, is_original=True)
146
+
147
+ raise HTTPError(response.status_code)
148
+
149
+ try:
150
+ message_id = int(response.content.get("id"))
151
+ except (AttributeError, ValueError):
152
+ message_id = 0
153
+
154
+ return message_id
155
+
156
+
157
+ def _make_key_last_request(url):
158
+ return f"killtracker-webhook-last-request-{url}"
159
+
160
+
161
+ def _make_key_retry_at(url):
162
+ return f"killtracker-webhook-retry-at-{url}"
@@ -0,0 +1,13 @@
1
+ """Helper functions for core modules."""
2
+
3
+ import datetime as dt
4
+ from typing import Any, Optional
5
+
6
+
7
+ def datetime_or_none(v: Any) -> Optional[dt.datetime]:
8
+ """Returns as datetime when it is a datetime else returns None."""
9
+ if v is None:
10
+ return None
11
+ if not isinstance(v, dt.datetime):
12
+ return None
13
+ return v
@@ -1,14 +1,12 @@
1
- """This module allows to create Discord messages from killmails."""
1
+ """Generate Discord messages from tracked killmails."""
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- import json
6
- from copy import copy
7
5
  from dataclasses import dataclass
8
- from typing import TYPE_CHECKING, List, Optional
6
+ from typing import TYPE_CHECKING, Optional
9
7
 
10
8
  import dhooks_lite
11
- from requests.exceptions import HTTPError
9
+ import requests
12
10
 
13
11
  from eveuniverse.helpers import EveEntityNameResolver
14
12
  from eveuniverse.models import EveEntity, EveSolarSystem
@@ -16,89 +14,32 @@ from eveuniverse.models import EveEntity, EveSolarSystem
16
14
  from allianceauth.eveonline.evelinks import dotlan, eveimageserver, zkillboard
17
15
  from allianceauth.services.hooks import get_extension_logger
18
16
  from app_utils.django import app_labels
19
- from app_utils.json import JSONDateTimeDecoder, JSONDateTimeEncoder
20
17
  from app_utils.logging import LoggerAddTag
21
18
  from app_utils.urls import static_file_absolute_url
22
19
  from app_utils.views import humanize_value
23
20
 
24
21
  from killtracker import __title__
25
-
26
- from .killmails import ZKB_KILLMAIL_BASEURL, Killmail, TrackerInfo
22
+ from killtracker.core.discord import DiscordMessage
23
+ from killtracker.core.zkb import ZKB_KILLMAIL_BASEURL, Killmail, TrackerInfo
27
24
 
28
25
  if TYPE_CHECKING:
29
26
  from killtracker.models import Tracker
30
27
 
31
-
32
28
  _ICON_SIZE = 128
33
29
 
34
-
35
30
  logger = LoggerAddTag(get_extension_logger(__name__), __title__)
36
31
 
37
32
 
38
- @dataclass
39
- class DiscordMessage:
40
- """A Discord message created from a Killmail."""
41
-
42
- avatar_url: Optional[str] = None
43
- content: Optional[str] = None
44
- embeds: Optional[List[dhooks_lite.Embed]] = None
45
- killmail_id: int = 0 # Killmail ID this message from created from
46
- tts: Optional[bool] = None
47
- username: Optional[str] = None
48
-
49
- def __post_init__(self):
50
- if not self.content and not self.embeds:
51
- raise ValueError("Message must have content or embeds to be valid")
52
-
53
- def to_json(self) -> str:
54
- """Converts a Discord message into a JSON object and returns it."""
55
-
56
- if self.embeds:
57
- embeds_list = [obj.asdict() for obj in self.embeds]
58
- else:
59
- embeds_list = None
60
-
61
- message = {}
62
- if self.killmail_id:
63
- message["killmail_id"] = self.killmail_id
64
- if self.content:
65
- message["content"] = self.content
66
- if embeds_list:
67
- message["embeds"] = embeds_list
68
- if self.tts:
69
- message["tts"] = self.tts
70
- if self.username:
71
- message["username"] = self.username
72
- if self.avatar_url:
73
- message["avatar_url"] = self.avatar_url
74
-
75
- return json.dumps(message, cls=JSONDateTimeEncoder)
76
-
77
- @classmethod
78
- def from_json(cls, s: str) -> "DiscordMessage":
79
- """Creates a DiscordMessage object from an JSON object and returns it."""
80
- message1: dict = json.loads(s, cls=JSONDateTimeDecoder)
81
- message2 = copy(message1)
82
- if message1.get("embeds"):
83
- message2["embeds"] = [
84
- dhooks_lite.Embed.from_dict(embed_dict)
85
- for embed_dict in message1.get("embeds")
86
- ]
87
- else:
88
- message2["embeds"] = None
89
- return cls(**message2)
90
-
91
- @classmethod
92
- def from_killmail(
93
- cls, tracker: Tracker, killmail: Killmail, intro_text: Optional[str] = None
94
- ) -> "DiscordMessage":
95
- """Creates a DiscordMessage object from a Killmail and returns it."""
96
- m = DiscordMessage(
97
- killmail_id=killmail.id,
98
- content=_create_content(tracker, intro_text),
99
- embeds=[_create_embed(tracker, killmail)],
100
- )
101
- return m
33
+ def create_discord_message_from_killmail(
34
+ tracker: Tracker, killmail: Killmail, intro_text: Optional[str] = None
35
+ ) -> "DiscordMessage":
36
+ """Creates a Discord message from a Killmail and returns it."""
37
+ m = DiscordMessage(
38
+ killmail_id=killmail.id,
39
+ content=_create_content(tracker, intro_text),
40
+ embeds=[_create_embed(tracker, killmail)],
41
+ )
42
+ return m
102
43
 
103
44
 
104
45
  @dataclass(frozen=True)
@@ -154,7 +95,7 @@ def _create_content(tracker: Tracker, intro_text: Optional[str] = None) -> str:
154
95
  for group in tracker.ping_groups.all():
155
96
  try:
156
97
  role = DiscordUser.objects.group_to_role(group) # type: ignore
157
- except HTTPError:
98
+ except requests.exceptions.HTTPError:
158
99
  logger.warning(
159
100
  "Failed to get Discord roles. Can not ping groups.",
160
101
  exc_info=True,
@@ -1,5 +1,4 @@
1
- """Module worker_shutdown allows tasks to find out
2
- whether their worker is currently shutting down.
1
+ """Allows tasks to find out whether their worker is currently shutting down.
3
2
 
4
3
  This enables long running tasks to abort early,
5
4
  which helps to speed up a warm worker shutdown.
@@ -19,12 +18,12 @@ _TIMEOUT_SECONDS = 120
19
18
  logger = LoggerAddTag(get_extension_logger(__name__), __title__)
20
19
 
21
20
 
22
- def reset(hostname: str) -> None:
21
+ def state_reset(hostname: str) -> None:
23
22
  """Resets the shutting down state for a worker."""
24
23
  cache.delete(_make_key(hostname))
25
24
 
26
25
 
27
- def set(hostname: str) -> None:
26
+ def state_set(hostname: str) -> None:
28
27
  """Sets a worker into the shutting down state."""
29
28
  cache.set(_make_key(hostname), "shutting down", timeout=_TIMEOUT_SECONDS)
30
29
 
@@ -1,4 +1,4 @@
1
- """Fetching killmails from ZKB."""
1
+ """Fetch killmails from zKillboard."""
2
2
 
3
3
  # pylint: disable = redefined-builtin
4
4
 
@@ -32,19 +32,22 @@ from killtracker.app_settings import (
32
32
  KILLTRACKER_STORAGE_KILLMAILS_LIFETIME,
33
33
  KILLTRACKER_ZKB_REQUEST_DELAY,
34
34
  )
35
- from killtracker.exceptions import KillmailDoesNotExist
35
+ from killtracker.core.helpers import datetime_or_none
36
36
  from killtracker.providers import esi
37
37
 
38
- logger = LoggerAddTag(get_extension_logger(__name__), __title__)
39
-
40
- ZKB_REDISQ_URL = "https://zkillredisq.stream/listen.php"
41
- ZKB_API_URL = "https://zkillboard.com/api/"
42
38
  ZKB_KILLMAIL_BASEURL = "https://zkillboard.com/kill/"
43
- REQUESTS_TIMEOUT = (5, 30)
44
- DEFAULT_429_TIMEOUT = 10
45
39
 
46
- MAIN_MINIMUM_COUNT = 2
47
- MAIN_MINIMUM_SHARE = 0.25
40
+ _KEY_RETRY_AT = "killtracker-zkb-retry-at"
41
+ _KEY_LAST_REQUEST = "killtracker-zkb-last-request"
42
+ _MAIN_MINIMUM_COUNT = 2
43
+ _MAIN_MINIMUM_SHARE = 0.25
44
+ _REQUESTS_TIMEOUT = (5, 30)
45
+ _ZKB_429_DEFAULT_TIMEOUT = 10
46
+ _ZKB_API_URL = "https://zkillboard.com/api/"
47
+ _ZKB_REDISQ_URL = "https://zkillredisq.stream/listen.php"
48
+
49
+ logger = LoggerAddTag(get_extension_logger(__name__), __title__)
50
+
48
51
 
49
52
  # TODO: Factor out logic for accessing the API to another module
50
53
 
@@ -57,6 +60,10 @@ class ZKBTooManyRequestsError(Exception):
57
60
  self.is_original = is_original
58
61
 
59
62
 
63
+ class KillmailDoesNotExist(Exception):
64
+ """Killmail does not exist in storage."""
65
+
66
+
60
67
  @dataclass
61
68
  class _KillmailBase:
62
69
  """Base class for all Killmail."""
@@ -264,8 +271,8 @@ class Killmail(_KillmailBase):
264
271
  jumps: Optional[int] = None,
265
272
  distance: Optional[float] = None,
266
273
  matching_ship_type_ids: Optional[List[int]] = None,
267
- minimum_count: int = MAIN_MINIMUM_COUNT,
268
- minimum_share: float = MAIN_MINIMUM_SHARE,
274
+ minimum_count: int = _MAIN_MINIMUM_COUNT,
275
+ minimum_share: float = _MAIN_MINIMUM_SHARE,
269
276
  ) -> "Killmail":
270
277
  """Clone this killmail and add tracker info."""
271
278
  main_ship_group = self._calc_main_attacker_ship_group(
@@ -419,45 +426,30 @@ class Killmail(_KillmailBase):
419
426
  if "," in KILLTRACKER_QUEUE_ID:
420
427
  raise ImproperlyConfigured("A queue ID must not contains commas.")
421
428
 
422
- key_retry_at = "killtracker-retry-at"
423
- if (v := cache.get(key_retry_at)) is not None:
424
- try:
425
- retry_at = dt.datetime.fromisoformat(v)
426
- except (TypeError, ValueError):
427
- cache.delete(key_retry_at)
428
- logger.warning("unable to parse timestamp of retry at")
429
- return None
430
-
431
- if retry_at > now():
432
- raise ZKBTooManyRequestsError(retry_at=retry_at, is_original=False)
433
-
434
- key_last_request = "killtracker-last-request"
435
- if (v := cache.get(key_last_request)) is not None:
436
- try:
437
- last_request = dt.datetime.fromisoformat(v)
438
- except (TypeError, ValueError):
439
- cache.delete(key_last_request)
440
- logger.warning("unable to parse timestamp of last request")
441
- last_request = now()
429
+ retry_at = datetime_or_none(cache.get(_KEY_RETRY_AT))
430
+ if retry_at is not None and retry_at > now():
431
+ raise ZKBTooManyRequestsError(retry_at=retry_at, is_original=False)
442
432
 
433
+ last_request = datetime_or_none(cache.get(_KEY_LAST_REQUEST))
434
+ if last_request is not None:
443
435
  next_slot = last_request + dt.timedelta(
444
436
  milliseconds=KILLTRACKER_ZKB_REQUEST_DELAY
445
437
  )
446
438
  seconds = (next_slot - now()).total_seconds()
447
439
  if seconds > 0:
448
- logger.debug("Waiting %f seconds to for next free slot", seconds)
440
+ logger.debug("ZKB API: Waiting %f seconds for next free slot", seconds)
449
441
  sleep(seconds)
450
442
 
451
443
  response = requests.get(
452
- ZKB_REDISQ_URL,
444
+ _ZKB_REDISQ_URL,
453
445
  params={
454
446
  "queueID": quote_plus(KILLTRACKER_QUEUE_ID),
455
447
  "ttw": KILLTRACKER_REDISQ_TTW,
456
448
  },
457
- timeout=REQUESTS_TIMEOUT,
449
+ timeout=_REQUESTS_TIMEOUT,
458
450
  headers={"User-Agent": USER_AGENT_TEXT},
459
451
  )
460
- cache.set(key_last_request, dt.datetime.isoformat(now()), timeout=30)
452
+ cache.set(_KEY_LAST_REQUEST, now(), timeout=KILLTRACKER_ZKB_REQUEST_DELAY + 30)
461
453
  logger.debug(
462
454
  "Response from ZKB API: %d %s %s",
463
455
  response.status_code,
@@ -473,9 +465,9 @@ class Killmail(_KillmailBase):
473
465
  try:
474
466
  retry_after = int(response.headers["Retry-After"])
475
467
  except KeyError:
476
- retry_after = DEFAULT_429_TIMEOUT
468
+ retry_after = _ZKB_429_DEFAULT_TIMEOUT
477
469
  retry_at = now() + dt.timedelta(seconds=retry_after)
478
- cache.set(key_retry_at, retry_at.isoformat(), timeout=retry_after + 60)
470
+ cache.set(_KEY_RETRY_AT, retry_at, timeout=retry_after + 60)
479
471
  raise ZKBTooManyRequestsError(retry_at=retry_at, is_original=True)
480
472
 
481
473
  return None
@@ -514,9 +506,9 @@ class Killmail(_KillmailBase):
514
506
  "Trying to fetch killmail from ZKB API with killmail ID %d ...",
515
507
  killmail_id,
516
508
  )
517
- url = f"{ZKB_API_URL}killID/{killmail_id}/"
509
+ url = f"{_ZKB_API_URL}killID/{killmail_id}/"
518
510
  response = requests.get(
519
- url, timeout=REQUESTS_TIMEOUT, headers={"User-Agent": USER_AGENT_TEXT}
511
+ url, timeout=_REQUESTS_TIMEOUT, headers={"User-Agent": USER_AGENT_TEXT}
520
512
  )
521
513
  response.raise_for_status()
522
514
  zkb_data = response.json()
@@ -15,7 +15,7 @@ from app_utils.logging import LoggerAddTag
15
15
 
16
16
  from killtracker import __title__
17
17
  from killtracker.app_settings import KILLTRACKER_PURGE_KILLMAILS_AFTER_DAYS
18
- from killtracker.core.killmails import Killmail, _KillmailCharacter
18
+ from killtracker.core.zkb import Killmail, _KillmailCharacter
19
19
 
20
20
  logger = LoggerAddTag(get_extension_logger(__name__), __title__)
21
21
 
@@ -28,11 +28,10 @@ from app_utils.logging import LoggerAddTag
28
28
  from killtracker import __title__
29
29
  from killtracker.app_settings import KILLTRACKER_KILLMAIL_MAX_AGE_FOR_TRACKER
30
30
  from killtracker.constants import EveCategoryId, EveGroupId
31
- from killtracker.core.discord_messages import DiscordMessage
32
- from killtracker.core.killmails import Killmail
31
+ from killtracker.core.trackers import create_discord_message_from_killmail
32
+ from killtracker.core.zkb import Killmail
33
33
  from killtracker.managers import TrackerManager
34
-
35
- from .webhooks import Webhook
34
+ from killtracker.models.webhooks import Webhook
36
35
 
37
36
  logger = LoggerAddTag(get_extension_logger(__name__), __title__)
38
37
 
@@ -886,5 +885,5 @@ class Tracker(models.Model):
886
885
 
887
886
  Returns the new queue size.
888
887
  """
889
- message = DiscordMessage.from_killmail(self, killmail, intro_text)
888
+ message = create_discord_message_from_killmail(self, killmail, intro_text)
890
889
  return self.webhook.enqueue_message(message)