django-esi 8.0.0a4__py3-none-any.whl → 8.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of django-esi might be problematic. Click here for more details.
- {django_esi-8.0.0a4.dist-info → django_esi-8.0.0b2.dist-info}/METADATA +5 -3
- {django_esi-8.0.0a4.dist-info → django_esi-8.0.0b2.dist-info}/RECORD +48 -47
- esi/__init__.py +1 -1
- esi/aiopenapi3/plugins.py +99 -3
- esi/clients.py +56 -7
- esi/decorators.py +26 -10
- esi/exceptions.py +7 -3
- esi/helpers.py +38 -0
- esi/locale/cs_CZ/LC_MESSAGES/django.mo +0 -0
- esi/locale/cs_CZ/LC_MESSAGES/django.po +2 -2
- esi/locale/de/LC_MESSAGES/django.mo +0 -0
- esi/locale/de/LC_MESSAGES/django.po +2 -2
- esi/locale/en/LC_MESSAGES/django.mo +0 -0
- esi/locale/en/LC_MESSAGES/django.po +2 -2
- esi/locale/es/LC_MESSAGES/django.mo +0 -0
- esi/locale/es/LC_MESSAGES/django.po +2 -2
- esi/locale/fr_FR/LC_MESSAGES/django.mo +0 -0
- esi/locale/fr_FR/LC_MESSAGES/django.po +2 -2
- esi/locale/it_IT/LC_MESSAGES/django.mo +0 -0
- esi/locale/it_IT/LC_MESSAGES/django.po +2 -2
- esi/locale/ja/LC_MESSAGES/django.mo +0 -0
- esi/locale/ja/LC_MESSAGES/django.po +2 -2
- esi/locale/ko_KR/LC_MESSAGES/django.mo +0 -0
- esi/locale/ko_KR/LC_MESSAGES/django.po +2 -2
- esi/locale/nl_NL/LC_MESSAGES/django.mo +0 -0
- esi/locale/nl_NL/LC_MESSAGES/django.po +2 -2
- esi/locale/pl_PL/LC_MESSAGES/django.mo +0 -0
- esi/locale/pl_PL/LC_MESSAGES/django.po +2 -2
- esi/locale/ru/LC_MESSAGES/django.mo +0 -0
- esi/locale/ru/LC_MESSAGES/django.po +2 -2
- esi/locale/sk/LC_MESSAGES/django.mo +0 -0
- esi/locale/sk/LC_MESSAGES/django.po +2 -2
- esi/locale/uk/LC_MESSAGES/django.mo +0 -0
- esi/locale/uk/LC_MESSAGES/django.po +2 -2
- esi/locale/zh_Hans/LC_MESSAGES/django.mo +0 -0
- esi/locale/zh_Hans/LC_MESSAGES/django.po +2 -2
- esi/managers.pyi +3 -0
- esi/openapi_clients.py +188 -44
- esi/rate_limiting.py +50 -21
- esi/signals.py +21 -0
- esi/stubs.pyi +9 -9
- esi/tests/__init__.py +33 -11
- esi/tests/test_clients.py +77 -19
- esi/tests/test_decorators.py +61 -1
- esi/tests/test_openapi.json +65 -2
- esi/tests/test_openapi.py +512 -18
- {django_esi-8.0.0a4.dist-info → django_esi-8.0.0b2.dist-info}/WHEEL +0 -0
- {django_esi-8.0.0a4.dist-info → django_esi-8.0.0b2.dist-info}/licenses/LICENSE +0 -0
esi/openapi_clients.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import pathlib
|
|
3
3
|
import warnings
|
|
4
|
-
|
|
4
|
+
import datetime as dt
|
|
5
5
|
from hashlib import md5
|
|
6
|
+
from timeit import default_timer
|
|
6
7
|
from typing import Any
|
|
7
8
|
|
|
8
9
|
from aiopenapi3 import OpenAPI, FileSystemLoader
|
|
@@ -10,6 +11,7 @@ from aiopenapi3._types import ResponseDataType, ResponseHeadersType
|
|
|
10
11
|
from aiopenapi3.errors import HTTPServerError as base_HTTPServerError
|
|
11
12
|
from aiopenapi3.errors import HTTPClientError as base_HTTPClientError
|
|
12
13
|
from aiopenapi3.request import OperationIndex, RequestBase
|
|
14
|
+
from esi.rate_limiting import ESIRateLimitBucket, ESIRateLimits, interval_to_seconds
|
|
13
15
|
from httpx import (
|
|
14
16
|
AsyncClient, Client, HTTPStatusError, RequestError, Response, Timeout,
|
|
15
17
|
)
|
|
@@ -23,16 +25,21 @@ from django.utils.text import slugify
|
|
|
23
25
|
|
|
24
26
|
from esi import app_settings
|
|
25
27
|
from esi.exceptions import HTTPClientError, HTTPServerError, HTTPNotModified
|
|
26
|
-
from esi.aiopenapi3.plugins import
|
|
28
|
+
from esi.aiopenapi3.plugins import (
|
|
29
|
+
Add304ContentType, DjangoESIInit, PatchCompatibilityDatePlugin,
|
|
30
|
+
Trim204ContentType, MinifySpec
|
|
31
|
+
)
|
|
27
32
|
from esi.exceptions import ESIErrorLimitException
|
|
28
33
|
from esi.models import Token
|
|
34
|
+
from esi.signals import esi_request_statistics
|
|
29
35
|
from esi.stubs import ESIClientStub
|
|
30
36
|
|
|
31
37
|
from . import __title__, __url__, __version__
|
|
38
|
+
from .helpers import pascal_case_string
|
|
32
39
|
|
|
33
40
|
logger = logging.getLogger(__name__)
|
|
34
41
|
|
|
35
|
-
ETAG_EXPIRY = 60*60*24*7 # 7 days
|
|
42
|
+
ETAG_EXPIRY = 60 * 60 * 24 * 7 # 7 days
|
|
36
43
|
|
|
37
44
|
|
|
38
45
|
def _time_to_expiry(expires_header: str) -> int:
|
|
@@ -43,8 +50,10 @@ def _time_to_expiry(expires_header: str) -> int:
|
|
|
43
50
|
int: The cache TTL in seconds
|
|
44
51
|
"""
|
|
45
52
|
try:
|
|
46
|
-
expires_dt = datetime.strptime(str(expires_header), '%a, %d %b %Y %H:%M:%S %Z')
|
|
47
|
-
|
|
53
|
+
expires_dt = dt.datetime.strptime(str(expires_header), '%a, %d %b %Y %H:%M:%S %Z')
|
|
54
|
+
if expires_dt.tzinfo is None:
|
|
55
|
+
expires_dt = expires_dt.replace(tzinfo=dt.timezone.utc)
|
|
56
|
+
return max(int((expires_dt - dt.datetime.now(dt.timezone.utc)).total_seconds()), 0)
|
|
48
57
|
except ValueError:
|
|
49
58
|
return 0
|
|
50
59
|
|
|
@@ -74,7 +83,7 @@ def http_retry_sync() -> Retrying:
|
|
|
74
83
|
)
|
|
75
84
|
|
|
76
85
|
|
|
77
|
-
async def http_retry_async() -> AsyncRetrying:
|
|
86
|
+
async def http_retry_async() -> AsyncRetrying: # pragma: no cover
|
|
78
87
|
return AsyncRetrying(
|
|
79
88
|
retry=retry_if_exception(_httpx_exceptions),
|
|
80
89
|
wait=wait_combine(
|
|
@@ -85,13 +94,19 @@ async def http_retry_async() -> AsyncRetrying:
|
|
|
85
94
|
)
|
|
86
95
|
|
|
87
96
|
|
|
88
|
-
def _load_plugins(app_name):
|
|
97
|
+
def _load_plugins(app_name, tags: list[str] = [], operations: list[str] = []):
|
|
89
98
|
"""Load the plugins to make ESI work with this lib.
|
|
90
99
|
|
|
91
100
|
Args:
|
|
92
101
|
app_name (str): app name to use for internal etags
|
|
93
102
|
"""
|
|
94
|
-
return [
|
|
103
|
+
return [
|
|
104
|
+
PatchCompatibilityDatePlugin(),
|
|
105
|
+
Trim204ContentType(),
|
|
106
|
+
Add304ContentType(),
|
|
107
|
+
DjangoESIInit(app_name),
|
|
108
|
+
MinifySpec(tags, operations)
|
|
109
|
+
]
|
|
95
110
|
|
|
96
111
|
|
|
97
112
|
def _load_aiopenapi_client_sync(
|
|
@@ -100,7 +115,9 @@ def _load_aiopenapi_client_sync(
|
|
|
100
115
|
app_name: str,
|
|
101
116
|
user_agent: str,
|
|
102
117
|
tenant: str,
|
|
103
|
-
spec_file: str | None = None
|
|
118
|
+
spec_file: str | None = None,
|
|
119
|
+
tags: list[str] = [],
|
|
120
|
+
operations: list[str] = []) -> OpenAPI:
|
|
104
121
|
"""Create an OpenAPI3 Client from Spec
|
|
105
122
|
|
|
106
123
|
Args:
|
|
@@ -140,14 +157,14 @@ def _load_aiopenapi_client_sync(
|
|
|
140
157
|
session_factory=session_factory,
|
|
141
158
|
loader=FileSystemLoader(pathlib.Path(spec_file)),
|
|
142
159
|
use_operation_tags=True,
|
|
143
|
-
plugins=_load_plugins(app_name)
|
|
160
|
+
plugins=_load_plugins(app_name, tags, operations)
|
|
144
161
|
)
|
|
145
162
|
else:
|
|
146
163
|
return OpenAPI.load_sync(
|
|
147
164
|
url=spec_url,
|
|
148
165
|
session_factory=session_factory,
|
|
149
166
|
use_operation_tags=True,
|
|
150
|
-
plugins=_load_plugins(app_name)
|
|
167
|
+
plugins=_load_plugins(app_name, tags, operations)
|
|
151
168
|
)
|
|
152
169
|
|
|
153
170
|
|
|
@@ -157,7 +174,7 @@ async def _load_aiopenapi_client_async(
|
|
|
157
174
|
app_name: str,
|
|
158
175
|
user_agent: str,
|
|
159
176
|
tenant: str,
|
|
160
|
-
spec_file: str | None = None) -> OpenAPI:
|
|
177
|
+
spec_file: str | None = None) -> OpenAPI: # pragma: no cover
|
|
161
178
|
"""Create an OpenAPI3 Client from Spec Async
|
|
162
179
|
|
|
163
180
|
Args:
|
|
@@ -218,10 +235,15 @@ def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None = Non
|
|
|
218
235
|
Returns:
|
|
219
236
|
str: User-Agent string
|
|
220
237
|
"""
|
|
238
|
+
|
|
239
|
+
# Enforce PascalCase for `ua_appname` and strip whitespace
|
|
240
|
+
sanitized_ua_appname = pascal_case_string(ua_appname)
|
|
241
|
+
sanitized_appname = pascal_case_string(__title__)
|
|
242
|
+
|
|
221
243
|
return (
|
|
222
|
-
f"{
|
|
244
|
+
f"{sanitized_ua_appname}/{ua_version} "
|
|
223
245
|
f"({app_settings.ESI_USER_CONTACT_EMAIL}{f'; +{ua_url})' if ua_url else ')'} "
|
|
224
|
-
f"{
|
|
246
|
+
f"{sanitized_appname}/{__version__} (+{__url__})"
|
|
225
247
|
)
|
|
226
248
|
|
|
227
249
|
|
|
@@ -234,6 +256,7 @@ def esi_client_factory_sync(
|
|
|
234
256
|
ua_appname: str, ua_version: str, ua_url: str | None = None,
|
|
235
257
|
spec_file: str | None = None,
|
|
236
258
|
tenant: str = "tranquility",
|
|
259
|
+
tags: list[str] = [], operations: list[str] = [],
|
|
237
260
|
**kwargs) -> OpenAPI:
|
|
238
261
|
"""Generate a new OpenAPI ESI client.
|
|
239
262
|
Args:
|
|
@@ -248,7 +271,16 @@ def esi_client_factory_sync(
|
|
|
248
271
|
"""
|
|
249
272
|
user_agent = _build_user_agent(ua_appname, ua_version, ua_url)
|
|
250
273
|
spec_url = _get_spec_url()
|
|
251
|
-
return _load_aiopenapi_client_sync(
|
|
274
|
+
return _load_aiopenapi_client_sync(
|
|
275
|
+
spec_url,
|
|
276
|
+
compatibility_date,
|
|
277
|
+
ua_appname,
|
|
278
|
+
user_agent,
|
|
279
|
+
tenant,
|
|
280
|
+
spec_file,
|
|
281
|
+
tags,
|
|
282
|
+
operations
|
|
283
|
+
)
|
|
252
284
|
|
|
253
285
|
|
|
254
286
|
async def esi_client_factory_async(
|
|
@@ -256,7 +288,7 @@ async def esi_client_factory_async(
|
|
|
256
288
|
ua_appname: str, ua_version: str, ua_url: str | None = None,
|
|
257
289
|
spec_file: str | None = None,
|
|
258
290
|
tenant: str = "tranquility",
|
|
259
|
-
**kwargs) -> OpenAPI:
|
|
291
|
+
**kwargs) -> OpenAPI: # pragma: no cover
|
|
260
292
|
"""Generate a new OpenAPI ESI client.
|
|
261
293
|
Args:
|
|
262
294
|
compatibility_date (str): "YYYY-MM-DD" The latest version of ESI your client is tested with
|
|
@@ -278,9 +310,13 @@ class BaseEsiOperation():
|
|
|
278
310
|
self.method, self.url, self.operation, self.extra = operation
|
|
279
311
|
self.api = api
|
|
280
312
|
self.token: Token | None = None
|
|
313
|
+
self.bucket: ESIRateLimitBucket | None = None
|
|
314
|
+
|
|
281
315
|
self._args = []
|
|
282
316
|
self._kwargs = {}
|
|
283
317
|
|
|
318
|
+
self._set_bucket()
|
|
319
|
+
|
|
284
320
|
def __call__(self, *args, **kwargs) -> "BaseEsiOperation":
|
|
285
321
|
self._args = args
|
|
286
322
|
self._kwargs = kwargs
|
|
@@ -343,7 +379,7 @@ class BaseEsiOperation():
|
|
|
343
379
|
str: Key
|
|
344
380
|
"""
|
|
345
381
|
# ignore the token this will break the cache
|
|
346
|
-
return f"{slugify(self.api.app_name)}_etag_{self._cache_key()}"
|
|
382
|
+
return f"{slugify(self.api.app_name)}_etag_{self._cache_key()}" # type: ignore app_name is added by a plugin
|
|
347
383
|
|
|
348
384
|
def _cache_key(self) -> str:
|
|
349
385
|
"""Generate a key name used to cache responses based on method, url, args, kwargs
|
|
@@ -366,7 +402,7 @@ class BaseEsiOperation():
|
|
|
366
402
|
"""
|
|
367
403
|
_body = self._kwargs.pop("body", None)
|
|
368
404
|
if _body and not getattr(self.operation, "requestBody", False):
|
|
369
|
-
raise ValueError("Request Body provided on endpoint with no request body
|
|
405
|
+
raise ValueError("Request Body provided on endpoint with no request body parameter.")
|
|
370
406
|
return _body
|
|
371
407
|
|
|
372
408
|
def _extract_token_param(self) -> Token | None:
|
|
@@ -393,7 +429,7 @@ class BaseEsiOperation():
|
|
|
393
429
|
"""
|
|
394
430
|
return any(p.name == "before" or p.name == "after" for p in self.operation.parameters)
|
|
395
431
|
|
|
396
|
-
def _get_cache(self, cache_key: str, etag: str) -> tuple[ResponseHeadersType | None, Any, Response | None]:
|
|
432
|
+
def _get_cache(self, cache_key: str, etag: str | None) -> tuple[ResponseHeadersType | None, Any, Response | None]:
|
|
397
433
|
"""Retrieve cached response and validate expiry
|
|
398
434
|
Args:
|
|
399
435
|
cache_key (str): The cache key to retrieve
|
|
@@ -421,8 +457,13 @@ class BaseEsiOperation():
|
|
|
421
457
|
|
|
422
458
|
# check if etag is same before building models from cache
|
|
423
459
|
if etag:
|
|
424
|
-
if cached_response.headers.get('
|
|
460
|
+
if cached_response.headers.get('ETag') == etag:
|
|
425
461
|
# refresh/store the etag's TTL
|
|
462
|
+
self._send_signal(
|
|
463
|
+
status_code=0, # this is a cached response less a 304
|
|
464
|
+
headers=cached_response.headers,
|
|
465
|
+
latency=0
|
|
466
|
+
)
|
|
426
467
|
self._store_etag(cached_response.headers)
|
|
427
468
|
raise HTTPNotModified(
|
|
428
469
|
status_code=304,
|
|
@@ -493,8 +534,40 @@ class BaseEsiOperation():
|
|
|
493
534
|
)
|
|
494
535
|
return req._process_request(cached_response)
|
|
495
536
|
|
|
537
|
+
def _set_bucket(self):
|
|
538
|
+
"""Setup the rate bucket"""
|
|
539
|
+
_rate_limit = getattr(self.operation, "extensions", {}).get("rate-limit", False)
|
|
540
|
+
if _rate_limit:
|
|
541
|
+
_key = _rate_limit["group"]
|
|
542
|
+
if self.token:
|
|
543
|
+
_key = f"{_key}:{self.token.character_id}"
|
|
544
|
+
self.bucket = ESIRateLimitBucket(
|
|
545
|
+
_key,
|
|
546
|
+
_rate_limit["max-tokens"],
|
|
547
|
+
interval_to_seconds(_rate_limit["window-size"])
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
def _send_signal(self, status_code: int, headers: dict = {}, latency: float = 0) -> None:
|
|
551
|
+
"""
|
|
552
|
+
Dispatch the esi request statistics signal
|
|
553
|
+
"""
|
|
554
|
+
esi_request_statistics.send(
|
|
555
|
+
sender=self.__class__,
|
|
556
|
+
operation=self.operation.operationId,
|
|
557
|
+
status_code=status_code,
|
|
558
|
+
headers=headers,
|
|
559
|
+
latency=latency,
|
|
560
|
+
bucket=self.bucket.slug if self.bucket else ""
|
|
561
|
+
)
|
|
562
|
+
|
|
496
563
|
|
|
497
564
|
class EsiOperation(BaseEsiOperation):
|
|
565
|
+
def __skip__process__headers__(
|
|
566
|
+
self, result, headers: dict[str, str], expected_response
|
|
567
|
+
):
|
|
568
|
+
"""Return all headers always"""
|
|
569
|
+
return headers
|
|
570
|
+
|
|
498
571
|
def _make_request(
|
|
499
572
|
self,
|
|
500
573
|
parameters: dict[str, Any],
|
|
@@ -507,10 +580,20 @@ class EsiOperation(BaseEsiOperation):
|
|
|
507
580
|
# or handle this by pushing their celery tasks back
|
|
508
581
|
raise ESIErrorLimitException(reset=reset)
|
|
509
582
|
|
|
583
|
+
if self.bucket:
|
|
584
|
+
"""Check Rate Limit"""
|
|
585
|
+
ESIRateLimits.check_bucket(self.bucket)
|
|
586
|
+
|
|
510
587
|
retry = http_retry_sync()
|
|
511
588
|
|
|
512
589
|
def __func():
|
|
513
590
|
req = self.api.createRequest(f"{self.operation.tags[0]}.{self.operation.operationId}")
|
|
591
|
+
|
|
592
|
+
# We want all headers from ESI
|
|
593
|
+
# don't check/parse them against the spec and return them all
|
|
594
|
+
# TODO Investigate if this is a bug with aiopenapi or a spec compliance issue
|
|
595
|
+
req._process__headers = self.__skip__process__headers__
|
|
596
|
+
|
|
514
597
|
if self.token:
|
|
515
598
|
self.api.authenticate(OAuth2=True) # make the lib happy
|
|
516
599
|
if isinstance(self.token, str):
|
|
@@ -529,7 +612,22 @@ class EsiOperation(BaseEsiOperation):
|
|
|
529
612
|
req.req.headers["Authorization"] = f"Bearer {self.token.valid_access_token()}"
|
|
530
613
|
if etag:
|
|
531
614
|
req.req.headers["If-None-Match"] = etag
|
|
532
|
-
|
|
615
|
+
_response = req.request(data=self.body, parameters=self._unnormalize_parameters(parameters))
|
|
616
|
+
|
|
617
|
+
if self.bucket and "x-ratelimit-remaining" in _response.result.headers:
|
|
618
|
+
logger.debug(
|
|
619
|
+
"ESI Rate-Limit: "
|
|
620
|
+
f"'{_response.result.headers.get('x-ratelimit-group')}' - "
|
|
621
|
+
f"Used {_response.result.headers.get('x-ratelimit-used')} - "
|
|
622
|
+
f"{_response.result.headers.get('x-ratelimit-remaining')} / "
|
|
623
|
+
f"({_response.result.headers.get('x-ratelimit-limit')})"
|
|
624
|
+
)
|
|
625
|
+
ESIRateLimits.set_bucket(
|
|
626
|
+
self.bucket,
|
|
627
|
+
_response.result.headers.get("x-ratelimit-remaining")
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
return _response
|
|
533
631
|
return retry(__func)
|
|
534
632
|
|
|
535
633
|
def result(
|
|
@@ -542,11 +640,16 @@ class EsiOperation(BaseEsiOperation):
|
|
|
542
640
|
"""Executes the request and returns the response from ESI for the current operation.
|
|
543
641
|
Raises:
|
|
544
642
|
ESIErrorLimitException: _description_
|
|
643
|
+
ESIBucketLimitException: _description_
|
|
545
644
|
Returns:
|
|
546
645
|
_type_: _description_
|
|
547
646
|
"""
|
|
548
|
-
|
|
647
|
+
_t = default_timer()
|
|
549
648
|
self.token = self._extract_token_param()
|
|
649
|
+
|
|
650
|
+
if self.token:
|
|
651
|
+
self._set_bucket()
|
|
652
|
+
|
|
550
653
|
self.body = self._extract_body_param()
|
|
551
654
|
parameters = self._kwargs | extra
|
|
552
655
|
cache_key = self._cache_key()
|
|
@@ -576,13 +679,13 @@ class EsiOperation(BaseEsiOperation):
|
|
|
576
679
|
logger.debug(f"Cache Miss {self.url}")
|
|
577
680
|
try:
|
|
578
681
|
headers, data, response = self._make_request(parameters, etag)
|
|
579
|
-
if response.status_code == 420:
|
|
580
|
-
reset = response.headers.get("X-RateLimit-Reset", None)
|
|
581
|
-
cache.set("esi_error_limit_reset", reset, timeout=reset)
|
|
582
|
-
raise ESIErrorLimitException(reset=reset)
|
|
583
|
-
|
|
584
682
|
# Shim our exceptions into Django-ESI
|
|
585
683
|
except base_HTTPServerError as e:
|
|
684
|
+
self._send_signal(
|
|
685
|
+
status_code=e.status_code,
|
|
686
|
+
headers=e.headers,
|
|
687
|
+
latency=default_timer() - _t
|
|
688
|
+
)
|
|
586
689
|
raise HTTPServerError(
|
|
587
690
|
status_code=e.status_code,
|
|
588
691
|
headers=e.headers,
|
|
@@ -590,23 +693,52 @@ class EsiOperation(BaseEsiOperation):
|
|
|
590
693
|
)
|
|
591
694
|
|
|
592
695
|
except base_HTTPClientError as e:
|
|
696
|
+
self._send_signal(
|
|
697
|
+
status_code=e.status_code,
|
|
698
|
+
headers=e.headers,
|
|
699
|
+
latency=default_timer() - _t
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
if e.status_code == 420:
|
|
703
|
+
reset = e.headers.get("X-RateLimit-Reset", None)
|
|
704
|
+
if reset:
|
|
705
|
+
reset = int(reset)
|
|
706
|
+
cache.set("esi_error_limit_reset", reset, timeout=reset)
|
|
707
|
+
raise ESIErrorLimitException(reset=reset)
|
|
708
|
+
|
|
593
709
|
raise HTTPClientError(
|
|
594
710
|
status_code=e.status_code,
|
|
595
711
|
headers=e.headers,
|
|
596
712
|
data=e.data
|
|
597
713
|
)
|
|
598
714
|
|
|
715
|
+
self._send_signal(
|
|
716
|
+
status_code=response.status_code,
|
|
717
|
+
headers=response.headers,
|
|
718
|
+
latency=default_timer() - _t
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
# store the ETAG in cache
|
|
722
|
+
self._store_etag(response.headers)
|
|
723
|
+
|
|
599
724
|
# Throw a 304 exception for catching.
|
|
600
725
|
if response.status_code == 304:
|
|
601
726
|
# refresh/store the etag's TTL
|
|
602
|
-
self._store_etag(response.headers)
|
|
603
727
|
raise HTTPNotModified(
|
|
604
728
|
status_code=304,
|
|
605
729
|
headers=response.headers
|
|
606
730
|
)
|
|
607
731
|
|
|
608
|
-
|
|
609
|
-
|
|
732
|
+
# last step store cache after 304 logic, we dont want to catch the 304 `None` responses
|
|
733
|
+
self._store_cache(cache_key, response)
|
|
734
|
+
|
|
735
|
+
else:
|
|
736
|
+
# send signal for cached data too
|
|
737
|
+
self._send_signal(
|
|
738
|
+
status_code=0,
|
|
739
|
+
headers=response.headers,
|
|
740
|
+
latency=default_timer() - _t
|
|
741
|
+
)
|
|
610
742
|
|
|
611
743
|
return (data, response) if return_response else data
|
|
612
744
|
|
|
@@ -724,7 +856,7 @@ class EsiOperation(BaseEsiOperation):
|
|
|
724
856
|
return []
|
|
725
857
|
|
|
726
858
|
|
|
727
|
-
class EsiOperationAsync(BaseEsiOperation):
|
|
859
|
+
class EsiOperationAsync(BaseEsiOperation): # pragma: no cover
|
|
728
860
|
async def _make_request(
|
|
729
861
|
self,
|
|
730
862
|
parameters: dict[str, Any],
|
|
@@ -747,7 +879,7 @@ class EsiOperationAsync(BaseEsiOperation):
|
|
|
747
879
|
req.req.headers["Authorization"] = f"Bearer {self.token.valid_access_token()}"
|
|
748
880
|
if etag:
|
|
749
881
|
req.req.headers["If-None-Match"] = etag
|
|
750
|
-
return
|
|
882
|
+
return req.request(parameters=self._unnormalize_parameters(parameters))
|
|
751
883
|
# Should never be reached because AsyncRetrying always yields at least once
|
|
752
884
|
raise RuntimeError("Retry loop exited without performing a request")
|
|
753
885
|
|
|
@@ -766,7 +898,7 @@ class EsiOperationAsync(BaseEsiOperation):
|
|
|
766
898
|
if not etag and app_settings.ESI_CACHE_RESPONSE:
|
|
767
899
|
etag = cache.get(etag_key)
|
|
768
900
|
|
|
769
|
-
headers, data, response = self._get_cache(cache_key)
|
|
901
|
+
headers, data, response = self._get_cache(cache_key, etag)
|
|
770
902
|
|
|
771
903
|
if response and use_cache:
|
|
772
904
|
expiry = _time_to_expiry(str(headers.get('Expires')))
|
|
@@ -868,7 +1000,7 @@ class EsiOperationAsync(BaseEsiOperation):
|
|
|
868
1000
|
my_languages.append(lang)
|
|
869
1001
|
|
|
870
1002
|
return {
|
|
871
|
-
language: self.results(accept_language=language, **
|
|
1003
|
+
language: self.results(accept_language=language, **extra)
|
|
872
1004
|
for language in my_languages
|
|
873
1005
|
}
|
|
874
1006
|
|
|
@@ -906,7 +1038,7 @@ class ESITag:
|
|
|
906
1038
|
return EsiOperation(self._operations[name], self.api)
|
|
907
1039
|
|
|
908
1040
|
|
|
909
|
-
class ESITagAsync():
|
|
1041
|
+
class ESITagAsync(): # pragma: no cover
|
|
910
1042
|
"""
|
|
911
1043
|
Async API Tag Wrapper, providing access to Operations within a tag
|
|
912
1044
|
Assets, Characters, etc.
|
|
@@ -966,9 +1098,9 @@ class ESIClient(ESIClientStub):
|
|
|
966
1098
|
# old lib
|
|
967
1099
|
from django.core.cache import caches
|
|
968
1100
|
default_cache = caches['default']
|
|
969
|
-
_client = default_cache.get_master_client()
|
|
1101
|
+
_client = default_cache.get_master_client() # type: ignore
|
|
970
1102
|
|
|
971
|
-
keys = _client.keys(f":?:{slugify(self.api.app_name)}_etag_*")
|
|
1103
|
+
keys = _client.keys(f":?:{slugify(self.api.app_name)}_etag_*") # type: ignore app_name is added by a plugin
|
|
972
1104
|
if keys:
|
|
973
1105
|
deleted = _client.delete(*keys)
|
|
974
1106
|
|
|
@@ -977,7 +1109,7 @@ class ESIClient(ESIClientStub):
|
|
|
977
1109
|
return deleted
|
|
978
1110
|
|
|
979
1111
|
|
|
980
|
-
class ESIClientAsync(ESIClientStub):
|
|
1112
|
+
class ESIClientAsync(ESIClientStub): # pragma: no cover
|
|
981
1113
|
"""
|
|
982
1114
|
Async Base ESI Client, provides access to Tags Assets, Characters, etc.
|
|
983
1115
|
or Raw aiopenapi3 via sad smiley ._.
|
|
@@ -1007,6 +1139,8 @@ class ESIClientAsync(ESIClientStub):
|
|
|
1007
1139
|
|
|
1008
1140
|
class ESIClientProvider:
|
|
1009
1141
|
"""Class for providing a single ESI client instance for a whole app
|
|
1142
|
+
* Note that one of either `tags` or `operations` must be provided to reduce memory footprint of the client
|
|
1143
|
+
* When `DEBUG=False`, not supplying either will raise an AttributeError.
|
|
1010
1144
|
Args:
|
|
1011
1145
|
compatibility_date (str | date): The compatibility date for the ESI client.
|
|
1012
1146
|
ua_appname (str): Name of the App for generating a User-Agent,
|
|
@@ -1014,6 +1148,8 @@ class ESIClientProvider:
|
|
|
1014
1148
|
ua_url (str, Optional): URL To the Source Code or Documentation for generating a User-Agent,
|
|
1015
1149
|
spec_file (str, Optional): Absolute path to a OpenApi 3.1 spec file to load.
|
|
1016
1150
|
tenant (str, Optional): The ESI tenant to use (default: "tranquility").
|
|
1151
|
+
operations (list[str], Optional*): List of operations to filter the spec down.
|
|
1152
|
+
tags (list[str], Optional*): List of tags to filter the spec down.
|
|
1017
1153
|
Functions:
|
|
1018
1154
|
client(): ESIClient
|
|
1019
1155
|
client_async(): ESIClientAsync
|
|
@@ -1024,24 +1160,28 @@ class ESIClientProvider:
|
|
|
1024
1160
|
|
|
1025
1161
|
def __init__(
|
|
1026
1162
|
self,
|
|
1027
|
-
compatibility_date: str | date,
|
|
1163
|
+
compatibility_date: str | dt.date,
|
|
1028
1164
|
ua_appname: str,
|
|
1029
1165
|
ua_version: str,
|
|
1030
1166
|
ua_url: str | None = None,
|
|
1031
1167
|
spec_file: None | str = None,
|
|
1032
1168
|
tenant: str = "tranquility",
|
|
1169
|
+
operations: list[str] = [],
|
|
1170
|
+
tags: list[str] = [],
|
|
1033
1171
|
**kwargs
|
|
1034
1172
|
) -> None:
|
|
1035
|
-
if type(compatibility_date) is date:
|
|
1036
|
-
self._compatibility_date = self._date_to_string(compatibility_date)
|
|
1173
|
+
if type(compatibility_date) is dt.date:
|
|
1174
|
+
self._compatibility_date: str = self._date_to_string(compatibility_date)
|
|
1037
1175
|
else:
|
|
1038
|
-
self._compatibility_date = compatibility_date
|
|
1176
|
+
self._compatibility_date: str = str(compatibility_date)
|
|
1039
1177
|
self._ua_appname = ua_appname
|
|
1040
1178
|
self._ua_version = ua_version
|
|
1041
1179
|
self._ua_url = ua_url
|
|
1042
1180
|
self._spec_file = spec_file
|
|
1043
1181
|
self._tenant = tenant
|
|
1044
1182
|
self._kwargs = kwargs
|
|
1183
|
+
self._operations = operations
|
|
1184
|
+
self._tags = tags
|
|
1045
1185
|
|
|
1046
1186
|
@property
|
|
1047
1187
|
def client(self) -> ESIClient:
|
|
@@ -1053,12 +1193,14 @@ class ESIClientProvider:
|
|
|
1053
1193
|
ua_url=self._ua_url,
|
|
1054
1194
|
spec_file=self._spec_file,
|
|
1055
1195
|
tenant=self._tenant,
|
|
1196
|
+
operations=self._operations,
|
|
1197
|
+
tags=self._tags,
|
|
1056
1198
|
**self._kwargs)
|
|
1057
1199
|
self._client = ESIClient(api)
|
|
1058
1200
|
return self._client
|
|
1059
1201
|
|
|
1060
1202
|
@property
|
|
1061
|
-
async def client_async(self) -> ESIClientAsync:
|
|
1203
|
+
async def client_async(self) -> ESIClientAsync: # pragma: no cover
|
|
1062
1204
|
if self._client_async is None:
|
|
1063
1205
|
api = await esi_client_factory_async(
|
|
1064
1206
|
compatibility_date=self._compatibility_date,
|
|
@@ -1067,12 +1209,14 @@ class ESIClientProvider:
|
|
|
1067
1209
|
ua_url=self._ua_url,
|
|
1068
1210
|
spec_file=self._spec_file,
|
|
1069
1211
|
tenant=self._tenant,
|
|
1212
|
+
operations=self._operations,
|
|
1213
|
+
tags=self._tags,
|
|
1070
1214
|
**self._kwargs)
|
|
1071
1215
|
self._client_async = ESIClientAsync(api)
|
|
1072
1216
|
return self._client_async
|
|
1073
1217
|
|
|
1074
1218
|
@classmethod
|
|
1075
|
-
def _date_to_string(cls, compatibility_date: date) -> str:
|
|
1219
|
+
def _date_to_string(cls, compatibility_date: dt.date) -> str:
|
|
1076
1220
|
"""Turns a date object in a compatibility_date string"""
|
|
1077
1221
|
return f"{compatibility_date.year}-{compatibility_date.month:02}-{compatibility_date.day:02}"
|
|
1078
1222
|
|
esi/rate_limiting.py
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import time
|
|
2
3
|
from django.core.cache import cache
|
|
3
4
|
from esi.exceptions import ESIBucketLimitException
|
|
4
5
|
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
seconds_per_unit = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def interval_to_seconds(s):
|
|
12
|
+
return int(s[:-1]) * seconds_per_unit[s[-1]]
|
|
13
|
+
|
|
5
14
|
|
|
6
15
|
class ESIRateLimitBucket:
|
|
7
16
|
MARKET_DATA_HISTORY = ("market_data_history", 300, 60)
|
|
@@ -16,6 +25,9 @@ class ESIRateLimitBucket:
|
|
|
16
25
|
def choices(cls):
|
|
17
26
|
return [(bucket.slug, bucket.slug.replace("_", " ").title()) for bucket in cls]
|
|
18
27
|
|
|
28
|
+
def __str__(self):
|
|
29
|
+
return f"Rate Limit: {self.slug} - {self.limit} in {self.window}Seconds"
|
|
30
|
+
|
|
19
31
|
|
|
20
32
|
class ESIRateLimiter:
|
|
21
33
|
def __init__(self) -> None:
|
|
@@ -35,44 +47,61 @@ class ESIRateLimiter:
|
|
|
35
47
|
|
|
36
48
|
def get_bucket(self, bucket: ESIRateLimitBucket) -> int:
|
|
37
49
|
# get the value from the bucket
|
|
38
|
-
return
|
|
39
|
-
|
|
40
|
-
|
|
50
|
+
return int(
|
|
51
|
+
cache.get(
|
|
52
|
+
self._slug_to_key(bucket.slug),
|
|
53
|
+
1 # When not found return 1
|
|
54
|
+
)
|
|
41
55
|
)
|
|
42
56
|
|
|
43
|
-
def get_timeout(self, bucket: ESIRateLimitBucket
|
|
44
|
-
|
|
45
|
-
if
|
|
57
|
+
def get_timeout(self, bucket: ESIRateLimitBucket) -> int:
|
|
58
|
+
current_bucket = self.get_bucket(bucket)
|
|
59
|
+
if current_bucket <= 0:
|
|
46
60
|
timeout = cache.ttl(self._slug_to_key(bucket.slug)) + 1
|
|
47
61
|
msg = (
|
|
48
62
|
f"Rate limit for bucket '{bucket.slug}' exceeded: "
|
|
49
|
-
f"{
|
|
63
|
+
f"{current_bucket}/{bucket.limit} in last {bucket.window}s. "
|
|
50
64
|
f"Wait {timeout}s."
|
|
51
65
|
)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
else:
|
|
55
|
-
return timeout # return the time left till reset
|
|
66
|
+
logger.warning(msg)
|
|
67
|
+
return timeout # return the time left till reset
|
|
56
68
|
else:
|
|
57
69
|
return 0 # we are good.
|
|
58
70
|
|
|
59
|
-
def decr_bucket(self, bucket: ESIRateLimitBucket) -> int:
|
|
60
|
-
# decrease the bucket value by
|
|
71
|
+
def decr_bucket(self, bucket: ESIRateLimitBucket, delta: int = 1) -> int:
|
|
72
|
+
# decrease the bucket value by <delta> from the bucket
|
|
61
73
|
return cache.decr(
|
|
62
|
-
self._slug_to_key(bucket.slug)
|
|
74
|
+
self._slug_to_key(bucket.slug),
|
|
75
|
+
delta
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def set_bucket(self, bucket: ESIRateLimitBucket, new_limit: int = 1) -> int:
|
|
79
|
+
# decrease the bucket value by <delta> from the bucket
|
|
80
|
+
return cache.set(
|
|
81
|
+
self._slug_to_key(bucket.slug),
|
|
82
|
+
int(new_limit),
|
|
83
|
+
timeout=bucket.window
|
|
63
84
|
)
|
|
64
85
|
|
|
65
|
-
def check_bucket(self, bucket: ESIRateLimitBucket
|
|
66
|
-
|
|
86
|
+
def check_bucket(self, bucket: ESIRateLimitBucket):
|
|
87
|
+
self.init_bucket(bucket)
|
|
67
88
|
# get the value
|
|
68
|
-
bucket_val =
|
|
89
|
+
bucket_val = self.get_bucket(bucket)
|
|
69
90
|
if bucket_val <= 0:
|
|
70
|
-
timeout =
|
|
91
|
+
timeout = self.get_timeout(bucket)
|
|
71
92
|
if timeout > 0:
|
|
72
|
-
|
|
93
|
+
raise ESIBucketLimitException(bucket, timeout)
|
|
73
94
|
return
|
|
74
|
-
|
|
75
|
-
|
|
95
|
+
|
|
96
|
+
def check_decr_bucket(self, bucket: ESIRateLimitBucket, raise_on_limit: bool = True):
|
|
97
|
+
try:
|
|
98
|
+
self.check_bucket(bucket)
|
|
99
|
+
self.decr_bucket(bucket)
|
|
100
|
+
except ESIBucketLimitException as ex:
|
|
101
|
+
if raise_on_limit:
|
|
102
|
+
raise ex
|
|
103
|
+
else:
|
|
104
|
+
time.sleep(ex.reset)
|
|
76
105
|
|
|
77
106
|
|
|
78
107
|
ESIRateLimits = ESIRateLimiter()
|
esi/signals.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from django.dispatch import Signal
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Basic signal sent after an ESI requests
|
|
5
|
+
from django.dispatch import receiver
|
|
6
|
+
from esi.signals import esi_request_statistics
|
|
7
|
+
|
|
8
|
+
@receiver(esi_request_statistics)
|
|
9
|
+
def esi_callback(sender, operation, status_code, headers, latency, bucket, **kwargs):
|
|
10
|
+
# do stuff
|
|
11
|
+
pass
|
|
12
|
+
"""
|
|
13
|
+
esi_request_statistics = Signal(
|
|
14
|
+
# providing_args=[
|
|
15
|
+
# "operation",
|
|
16
|
+
# "status_code",
|
|
17
|
+
# "headers",
|
|
18
|
+
# "latency",
|
|
19
|
+
# "bucket"
|
|
20
|
+
# ]
|
|
21
|
+
)
|