django-esi 8.0.0b1__py3-none-any.whl → 8.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of django-esi might be problematic. Click here for more details.
- {django_esi-8.0.0b1.dist-info → django_esi-8.0.0b2.dist-info}/METADATA +3 -2
- {django_esi-8.0.0b1.dist-info → django_esi-8.0.0b2.dist-info}/RECORD +46 -45
- esi/__init__.py +1 -1
- esi/aiopenapi3/plugins.py +12 -2
- esi/clients.py +41 -1
- esi/decorators.py +26 -10
- esi/exceptions.py +7 -3
- esi/helpers.py +1 -0
- esi/locale/cs_CZ/LC_MESSAGES/django.mo +0 -0
- esi/locale/cs_CZ/LC_MESSAGES/django.po +2 -2
- esi/locale/de/LC_MESSAGES/django.mo +0 -0
- esi/locale/de/LC_MESSAGES/django.po +2 -2
- esi/locale/en/LC_MESSAGES/django.mo +0 -0
- esi/locale/en/LC_MESSAGES/django.po +2 -2
- esi/locale/es/LC_MESSAGES/django.mo +0 -0
- esi/locale/es/LC_MESSAGES/django.po +2 -2
- esi/locale/fr_FR/LC_MESSAGES/django.mo +0 -0
- esi/locale/fr_FR/LC_MESSAGES/django.po +2 -2
- esi/locale/it_IT/LC_MESSAGES/django.mo +0 -0
- esi/locale/it_IT/LC_MESSAGES/django.po +2 -2
- esi/locale/ja/LC_MESSAGES/django.mo +0 -0
- esi/locale/ja/LC_MESSAGES/django.po +2 -2
- esi/locale/ko_KR/LC_MESSAGES/django.mo +0 -0
- esi/locale/ko_KR/LC_MESSAGES/django.po +2 -2
- esi/locale/nl_NL/LC_MESSAGES/django.mo +0 -0
- esi/locale/nl_NL/LC_MESSAGES/django.po +2 -2
- esi/locale/pl_PL/LC_MESSAGES/django.mo +0 -0
- esi/locale/pl_PL/LC_MESSAGES/django.po +2 -2
- esi/locale/ru/LC_MESSAGES/django.mo +0 -0
- esi/locale/ru/LC_MESSAGES/django.po +2 -2
- esi/locale/sk/LC_MESSAGES/django.mo +0 -0
- esi/locale/sk/LC_MESSAGES/django.po +2 -2
- esi/locale/uk/LC_MESSAGES/django.mo +0 -0
- esi/locale/uk/LC_MESSAGES/django.po +2 -2
- esi/locale/zh_Hans/LC_MESSAGES/django.mo +0 -0
- esi/locale/zh_Hans/LC_MESSAGES/django.po +2 -2
- esi/managers.pyi +3 -0
- esi/openapi_clients.py +139 -31
- esi/rate_limiting.py +50 -21
- esi/signals.py +21 -0
- esi/tests/__init__.py +30 -8
- esi/tests/test_decorators.py +61 -1
- esi/tests/test_openapi.json +65 -2
- esi/tests/test_openapi.py +387 -47
- {django_esi-8.0.0b1.dist-info → django_esi-8.0.0b2.dist-info}/WHEEL +0 -0
- {django_esi-8.0.0b1.dist-info → django_esi-8.0.0b2.dist-info}/licenses/LICENSE +0 -0
esi/openapi_clients.py
CHANGED
|
@@ -3,6 +3,7 @@ import pathlib
|
|
|
3
3
|
import warnings
|
|
4
4
|
import datetime as dt
|
|
5
5
|
from hashlib import md5
|
|
6
|
+
from timeit import default_timer
|
|
6
7
|
from typing import Any
|
|
7
8
|
|
|
8
9
|
from aiopenapi3 import OpenAPI, FileSystemLoader
|
|
@@ -10,6 +11,7 @@ from aiopenapi3._types import ResponseDataType, ResponseHeadersType
|
|
|
10
11
|
from aiopenapi3.errors import HTTPServerError as base_HTTPServerError
|
|
11
12
|
from aiopenapi3.errors import HTTPClientError as base_HTTPClientError
|
|
12
13
|
from aiopenapi3.request import OperationIndex, RequestBase
|
|
14
|
+
from esi.rate_limiting import ESIRateLimitBucket, ESIRateLimits, interval_to_seconds
|
|
13
15
|
from httpx import (
|
|
14
16
|
AsyncClient, Client, HTTPStatusError, RequestError, Response, Timeout,
|
|
15
17
|
)
|
|
@@ -29,6 +31,7 @@ from esi.aiopenapi3.plugins import (
|
|
|
29
31
|
)
|
|
30
32
|
from esi.exceptions import ESIErrorLimitException
|
|
31
33
|
from esi.models import Token
|
|
34
|
+
from esi.signals import esi_request_statistics
|
|
32
35
|
from esi.stubs import ESIClientStub
|
|
33
36
|
|
|
34
37
|
from . import __title__, __url__, __version__
|
|
@@ -36,7 +39,7 @@ from .helpers import pascal_case_string
|
|
|
36
39
|
|
|
37
40
|
logger = logging.getLogger(__name__)
|
|
38
41
|
|
|
39
|
-
ETAG_EXPIRY = 60*60*24*7 # 7 days
|
|
42
|
+
ETAG_EXPIRY = 60 * 60 * 24 * 7 # 7 days
|
|
40
43
|
|
|
41
44
|
|
|
42
45
|
def _time_to_expiry(expires_header: str) -> int:
|
|
@@ -80,7 +83,7 @@ def http_retry_sync() -> Retrying:
|
|
|
80
83
|
)
|
|
81
84
|
|
|
82
85
|
|
|
83
|
-
async def http_retry_async() -> AsyncRetrying:
|
|
86
|
+
async def http_retry_async() -> AsyncRetrying: # pragma: no cover
|
|
84
87
|
return AsyncRetrying(
|
|
85
88
|
retry=retry_if_exception(_httpx_exceptions),
|
|
86
89
|
wait=wait_combine(
|
|
@@ -91,7 +94,7 @@ async def http_retry_async() -> AsyncRetrying:
|
|
|
91
94
|
)
|
|
92
95
|
|
|
93
96
|
|
|
94
|
-
def _load_plugins(app_name, tags: list[str]=[], operations: list[str]=[]):
|
|
97
|
+
def _load_plugins(app_name, tags: list[str] = [], operations: list[str] = []):
|
|
95
98
|
"""Load the plugins to make ESI work with this lib.
|
|
96
99
|
|
|
97
100
|
Args:
|
|
@@ -171,7 +174,7 @@ async def _load_aiopenapi_client_async(
|
|
|
171
174
|
app_name: str,
|
|
172
175
|
user_agent: str,
|
|
173
176
|
tenant: str,
|
|
174
|
-
spec_file: str | None = None) -> OpenAPI:
|
|
177
|
+
spec_file: str | None = None) -> OpenAPI: # pragma: no cover
|
|
175
178
|
"""Create an OpenAPI3 Client from Spec Async
|
|
176
179
|
|
|
177
180
|
Args:
|
|
@@ -253,7 +256,7 @@ def esi_client_factory_sync(
|
|
|
253
256
|
ua_appname: str, ua_version: str, ua_url: str | None = None,
|
|
254
257
|
spec_file: str | None = None,
|
|
255
258
|
tenant: str = "tranquility",
|
|
256
|
-
tags: list[str]=[], operations: list[str]=[],
|
|
259
|
+
tags: list[str] = [], operations: list[str] = [],
|
|
257
260
|
**kwargs) -> OpenAPI:
|
|
258
261
|
"""Generate a new OpenAPI ESI client.
|
|
259
262
|
Args:
|
|
@@ -285,7 +288,7 @@ async def esi_client_factory_async(
|
|
|
285
288
|
ua_appname: str, ua_version: str, ua_url: str | None = None,
|
|
286
289
|
spec_file: str | None = None,
|
|
287
290
|
tenant: str = "tranquility",
|
|
288
|
-
**kwargs) -> OpenAPI:
|
|
291
|
+
**kwargs) -> OpenAPI: # pragma: no cover
|
|
289
292
|
"""Generate a new OpenAPI ESI client.
|
|
290
293
|
Args:
|
|
291
294
|
compatibility_date (str): "YYYY-MM-DD" The latest version of ESI your client is tested with
|
|
@@ -307,9 +310,13 @@ class BaseEsiOperation():
|
|
|
307
310
|
self.method, self.url, self.operation, self.extra = operation
|
|
308
311
|
self.api = api
|
|
309
312
|
self.token: Token | None = None
|
|
313
|
+
self.bucket: ESIRateLimitBucket | None = None
|
|
314
|
+
|
|
310
315
|
self._args = []
|
|
311
316
|
self._kwargs = {}
|
|
312
317
|
|
|
318
|
+
self._set_bucket()
|
|
319
|
+
|
|
313
320
|
def __call__(self, *args, **kwargs) -> "BaseEsiOperation":
|
|
314
321
|
self._args = args
|
|
315
322
|
self._kwargs = kwargs
|
|
@@ -372,7 +379,7 @@ class BaseEsiOperation():
|
|
|
372
379
|
str: Key
|
|
373
380
|
"""
|
|
374
381
|
# ignore the token this will break the cache
|
|
375
|
-
return f"{slugify(self.api.app_name)}_etag_{self._cache_key()}"
|
|
382
|
+
return f"{slugify(self.api.app_name)}_etag_{self._cache_key()}" # type: ignore app_name is added by a plugin
|
|
376
383
|
|
|
377
384
|
def _cache_key(self) -> str:
|
|
378
385
|
"""Generate a key name used to cache responses based on method, url, args, kwargs
|
|
@@ -395,7 +402,7 @@ class BaseEsiOperation():
|
|
|
395
402
|
"""
|
|
396
403
|
_body = self._kwargs.pop("body", None)
|
|
397
404
|
if _body and not getattr(self.operation, "requestBody", False):
|
|
398
|
-
raise ValueError("Request Body provided on endpoint with no request body
|
|
405
|
+
raise ValueError("Request Body provided on endpoint with no request body parameter.")
|
|
399
406
|
return _body
|
|
400
407
|
|
|
401
408
|
def _extract_token_param(self) -> Token | None:
|
|
@@ -422,7 +429,7 @@ class BaseEsiOperation():
|
|
|
422
429
|
"""
|
|
423
430
|
return any(p.name == "before" or p.name == "after" for p in self.operation.parameters)
|
|
424
431
|
|
|
425
|
-
def _get_cache(self, cache_key: str, etag: str) -> tuple[ResponseHeadersType | None, Any, Response | None]:
|
|
432
|
+
def _get_cache(self, cache_key: str, etag: str | None) -> tuple[ResponseHeadersType | None, Any, Response | None]:
|
|
426
433
|
"""Retrieve cached response and validate expiry
|
|
427
434
|
Args:
|
|
428
435
|
cache_key (str): The cache key to retrieve
|
|
@@ -452,6 +459,11 @@ class BaseEsiOperation():
|
|
|
452
459
|
if etag:
|
|
453
460
|
if cached_response.headers.get('ETag') == etag:
|
|
454
461
|
# refresh/store the etag's TTL
|
|
462
|
+
self._send_signal(
|
|
463
|
+
status_code=0, # this is a cached response less a 304
|
|
464
|
+
headers=cached_response.headers,
|
|
465
|
+
latency=0
|
|
466
|
+
)
|
|
455
467
|
self._store_etag(cached_response.headers)
|
|
456
468
|
raise HTTPNotModified(
|
|
457
469
|
status_code=304,
|
|
@@ -522,8 +534,40 @@ class BaseEsiOperation():
|
|
|
522
534
|
)
|
|
523
535
|
return req._process_request(cached_response)
|
|
524
536
|
|
|
537
|
+
def _set_bucket(self):
|
|
538
|
+
"""Setup the rate bucket"""
|
|
539
|
+
_rate_limit = getattr(self.operation, "extensions", {}).get("rate-limit", False)
|
|
540
|
+
if _rate_limit:
|
|
541
|
+
_key = _rate_limit["group"]
|
|
542
|
+
if self.token:
|
|
543
|
+
_key = f"{_key}:{self.token.character_id}"
|
|
544
|
+
self.bucket = ESIRateLimitBucket(
|
|
545
|
+
_key,
|
|
546
|
+
_rate_limit["max-tokens"],
|
|
547
|
+
interval_to_seconds(_rate_limit["window-size"])
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
def _send_signal(self, status_code: int, headers: dict = {}, latency: float = 0) -> None:
|
|
551
|
+
"""
|
|
552
|
+
Dispatch the esi request statistics signal
|
|
553
|
+
"""
|
|
554
|
+
esi_request_statistics.send(
|
|
555
|
+
sender=self.__class__,
|
|
556
|
+
operation=self.operation.operationId,
|
|
557
|
+
status_code=status_code,
|
|
558
|
+
headers=headers,
|
|
559
|
+
latency=latency,
|
|
560
|
+
bucket=self.bucket.slug if self.bucket else ""
|
|
561
|
+
)
|
|
562
|
+
|
|
525
563
|
|
|
526
564
|
class EsiOperation(BaseEsiOperation):
|
|
565
|
+
def __skip__process__headers__(
|
|
566
|
+
self, result, headers: dict[str, str], expected_response
|
|
567
|
+
):
|
|
568
|
+
"""Return all headers always"""
|
|
569
|
+
return headers
|
|
570
|
+
|
|
527
571
|
def _make_request(
|
|
528
572
|
self,
|
|
529
573
|
parameters: dict[str, Any],
|
|
@@ -536,10 +580,20 @@ class EsiOperation(BaseEsiOperation):
|
|
|
536
580
|
# or handle this by pushing their celery tasks back
|
|
537
581
|
raise ESIErrorLimitException(reset=reset)
|
|
538
582
|
|
|
583
|
+
if self.bucket:
|
|
584
|
+
"""Check Rate Limit"""
|
|
585
|
+
ESIRateLimits.check_bucket(self.bucket)
|
|
586
|
+
|
|
539
587
|
retry = http_retry_sync()
|
|
540
588
|
|
|
541
589
|
def __func():
|
|
542
590
|
req = self.api.createRequest(f"{self.operation.tags[0]}.{self.operation.operationId}")
|
|
591
|
+
|
|
592
|
+
# We want all headers from ESI
|
|
593
|
+
# don't check/parse them against the spec and return them all
|
|
594
|
+
# TODO Investigate if this is a bug with aiopenapi or a spec compliance issue
|
|
595
|
+
req._process__headers = self.__skip__process__headers__
|
|
596
|
+
|
|
543
597
|
if self.token:
|
|
544
598
|
self.api.authenticate(OAuth2=True) # make the lib happy
|
|
545
599
|
if isinstance(self.token, str):
|
|
@@ -558,7 +612,22 @@ class EsiOperation(BaseEsiOperation):
|
|
|
558
612
|
req.req.headers["Authorization"] = f"Bearer {self.token.valid_access_token()}"
|
|
559
613
|
if etag:
|
|
560
614
|
req.req.headers["If-None-Match"] = etag
|
|
561
|
-
|
|
615
|
+
_response = req.request(data=self.body, parameters=self._unnormalize_parameters(parameters))
|
|
616
|
+
|
|
617
|
+
if self.bucket and "x-ratelimit-remaining" in _response.result.headers:
|
|
618
|
+
logger.debug(
|
|
619
|
+
"ESI Rate-Limit: "
|
|
620
|
+
f"'{_response.result.headers.get('x-ratelimit-group')}' - "
|
|
621
|
+
f"Used {_response.result.headers.get('x-ratelimit-used')} - "
|
|
622
|
+
f"{_response.result.headers.get('x-ratelimit-remaining')} / "
|
|
623
|
+
f"({_response.result.headers.get('x-ratelimit-limit')})"
|
|
624
|
+
)
|
|
625
|
+
ESIRateLimits.set_bucket(
|
|
626
|
+
self.bucket,
|
|
627
|
+
_response.result.headers.get("x-ratelimit-remaining")
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
return _response
|
|
562
631
|
return retry(__func)
|
|
563
632
|
|
|
564
633
|
def result(
|
|
@@ -571,11 +640,16 @@ class EsiOperation(BaseEsiOperation):
|
|
|
571
640
|
"""Executes the request and returns the response from ESI for the current operation.
|
|
572
641
|
Raises:
|
|
573
642
|
ESIErrorLimitException: _description_
|
|
643
|
+
ESIBucketLimitException: _description_
|
|
574
644
|
Returns:
|
|
575
645
|
_type_: _description_
|
|
576
646
|
"""
|
|
577
|
-
|
|
647
|
+
_t = default_timer()
|
|
578
648
|
self.token = self._extract_token_param()
|
|
649
|
+
|
|
650
|
+
if self.token:
|
|
651
|
+
self._set_bucket()
|
|
652
|
+
|
|
579
653
|
self.body = self._extract_body_param()
|
|
580
654
|
parameters = self._kwargs | extra
|
|
581
655
|
cache_key = self._cache_key()
|
|
@@ -605,13 +679,13 @@ class EsiOperation(BaseEsiOperation):
|
|
|
605
679
|
logger.debug(f"Cache Miss {self.url}")
|
|
606
680
|
try:
|
|
607
681
|
headers, data, response = self._make_request(parameters, etag)
|
|
608
|
-
if response.status_code == 420:
|
|
609
|
-
reset = response.headers.get("X-RateLimit-Reset", None)
|
|
610
|
-
cache.set("esi_error_limit_reset", reset, timeout=reset)
|
|
611
|
-
raise ESIErrorLimitException(reset=reset)
|
|
612
|
-
|
|
613
682
|
# Shim our exceptions into Django-ESI
|
|
614
683
|
except base_HTTPServerError as e:
|
|
684
|
+
self._send_signal(
|
|
685
|
+
status_code=e.status_code,
|
|
686
|
+
headers=e.headers,
|
|
687
|
+
latency=default_timer() - _t
|
|
688
|
+
)
|
|
615
689
|
raise HTTPServerError(
|
|
616
690
|
status_code=e.status_code,
|
|
617
691
|
headers=e.headers,
|
|
@@ -619,24 +693,52 @@ class EsiOperation(BaseEsiOperation):
|
|
|
619
693
|
)
|
|
620
694
|
|
|
621
695
|
except base_HTTPClientError as e:
|
|
696
|
+
self._send_signal(
|
|
697
|
+
status_code=e.status_code,
|
|
698
|
+
headers=e.headers,
|
|
699
|
+
latency=default_timer() - _t
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
if e.status_code == 420:
|
|
703
|
+
reset = e.headers.get("X-RateLimit-Reset", None)
|
|
704
|
+
if reset:
|
|
705
|
+
reset = int(reset)
|
|
706
|
+
cache.set("esi_error_limit_reset", reset, timeout=reset)
|
|
707
|
+
raise ESIErrorLimitException(reset=reset)
|
|
708
|
+
|
|
622
709
|
raise HTTPClientError(
|
|
623
710
|
status_code=e.status_code,
|
|
624
711
|
headers=e.headers,
|
|
625
712
|
data=e.data
|
|
626
713
|
)
|
|
627
714
|
|
|
715
|
+
self._send_signal(
|
|
716
|
+
status_code=response.status_code,
|
|
717
|
+
headers=response.headers,
|
|
718
|
+
latency=default_timer() - _t
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
# store the ETAG in cache
|
|
722
|
+
self._store_etag(response.headers)
|
|
723
|
+
|
|
628
724
|
# Throw a 304 exception for catching.
|
|
629
725
|
if response.status_code == 304:
|
|
630
726
|
# refresh/store the etag's TTL
|
|
631
|
-
self._store_etag(response.headers)
|
|
632
727
|
raise HTTPNotModified(
|
|
633
728
|
status_code=304,
|
|
634
729
|
headers=response.headers
|
|
635
730
|
)
|
|
636
731
|
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
732
|
+
# last step store cache after 304 logic, we dont want to catch the 304 `None` responses
|
|
733
|
+
self._store_cache(cache_key, response)
|
|
734
|
+
|
|
735
|
+
else:
|
|
736
|
+
# send signal for cached data too
|
|
737
|
+
self._send_signal(
|
|
738
|
+
status_code=0,
|
|
739
|
+
headers=response.headers,
|
|
740
|
+
latency=default_timer() - _t
|
|
741
|
+
)
|
|
640
742
|
|
|
641
743
|
return (data, response) if return_response else data
|
|
642
744
|
|
|
@@ -754,7 +856,7 @@ class EsiOperation(BaseEsiOperation):
|
|
|
754
856
|
return []
|
|
755
857
|
|
|
756
858
|
|
|
757
|
-
class EsiOperationAsync(BaseEsiOperation):
|
|
859
|
+
class EsiOperationAsync(BaseEsiOperation): # pragma: no cover
|
|
758
860
|
async def _make_request(
|
|
759
861
|
self,
|
|
760
862
|
parameters: dict[str, Any],
|
|
@@ -777,7 +879,7 @@ class EsiOperationAsync(BaseEsiOperation):
|
|
|
777
879
|
req.req.headers["Authorization"] = f"Bearer {self.token.valid_access_token()}"
|
|
778
880
|
if etag:
|
|
779
881
|
req.req.headers["If-None-Match"] = etag
|
|
780
|
-
return
|
|
882
|
+
return req.request(parameters=self._unnormalize_parameters(parameters))
|
|
781
883
|
# Should never be reached because AsyncRetrying always yields at least once
|
|
782
884
|
raise RuntimeError("Retry loop exited without performing a request")
|
|
783
885
|
|
|
@@ -796,7 +898,7 @@ class EsiOperationAsync(BaseEsiOperation):
|
|
|
796
898
|
if not etag and app_settings.ESI_CACHE_RESPONSE:
|
|
797
899
|
etag = cache.get(etag_key)
|
|
798
900
|
|
|
799
|
-
headers, data, response = self._get_cache(cache_key)
|
|
901
|
+
headers, data, response = self._get_cache(cache_key, etag)
|
|
800
902
|
|
|
801
903
|
if response and use_cache:
|
|
802
904
|
expiry = _time_to_expiry(str(headers.get('Expires')))
|
|
@@ -898,7 +1000,7 @@ class EsiOperationAsync(BaseEsiOperation):
|
|
|
898
1000
|
my_languages.append(lang)
|
|
899
1001
|
|
|
900
1002
|
return {
|
|
901
|
-
language: self.results(accept_language=language, **
|
|
1003
|
+
language: self.results(accept_language=language, **extra)
|
|
902
1004
|
for language in my_languages
|
|
903
1005
|
}
|
|
904
1006
|
|
|
@@ -936,7 +1038,7 @@ class ESITag:
|
|
|
936
1038
|
return EsiOperation(self._operations[name], self.api)
|
|
937
1039
|
|
|
938
1040
|
|
|
939
|
-
class ESITagAsync():
|
|
1041
|
+
class ESITagAsync(): # pragma: no cover
|
|
940
1042
|
"""
|
|
941
1043
|
Async API Tag Wrapper, providing access to Operations within a tag
|
|
942
1044
|
Assets, Characters, etc.
|
|
@@ -996,9 +1098,9 @@ class ESIClient(ESIClientStub):
|
|
|
996
1098
|
# old lib
|
|
997
1099
|
from django.core.cache import caches
|
|
998
1100
|
default_cache = caches['default']
|
|
999
|
-
_client = default_cache.get_master_client()
|
|
1101
|
+
_client = default_cache.get_master_client() # type: ignore
|
|
1000
1102
|
|
|
1001
|
-
keys = _client.keys(f":?:{slugify(self.api.app_name)}_etag_*")
|
|
1103
|
+
keys = _client.keys(f":?:{slugify(self.api.app_name)}_etag_*") # type: ignore app_name is added by a plugin
|
|
1002
1104
|
if keys:
|
|
1003
1105
|
deleted = _client.delete(*keys)
|
|
1004
1106
|
|
|
@@ -1007,7 +1109,7 @@ class ESIClient(ESIClientStub):
|
|
|
1007
1109
|
return deleted
|
|
1008
1110
|
|
|
1009
1111
|
|
|
1010
|
-
class ESIClientAsync(ESIClientStub):
|
|
1112
|
+
class ESIClientAsync(ESIClientStub): # pragma: no cover
|
|
1011
1113
|
"""
|
|
1012
1114
|
Async Base ESI Client, provides access to Tags Assets, Characters, etc.
|
|
1013
1115
|
or Raw aiopenapi3 via sad smiley ._.
|
|
@@ -1037,6 +1139,8 @@ class ESIClientAsync(ESIClientStub):
|
|
|
1037
1139
|
|
|
1038
1140
|
class ESIClientProvider:
|
|
1039
1141
|
"""Class for providing a single ESI client instance for a whole app
|
|
1142
|
+
* Note that one of either `tags` or `operations` must be provided to reduce memory footprint of the client
|
|
1143
|
+
* When `DEBUG=False`, not supplying either will raise an AttributeError.
|
|
1040
1144
|
Args:
|
|
1041
1145
|
compatibility_date (str | date): The compatibility date for the ESI client.
|
|
1042
1146
|
ua_appname (str): Name of the App for generating a User-Agent,
|
|
@@ -1044,6 +1148,8 @@ class ESIClientProvider:
|
|
|
1044
1148
|
ua_url (str, Optional): URL To the Source Code or Documentation for generating a User-Agent,
|
|
1045
1149
|
spec_file (str, Optional): Absolute path to a OpenApi 3.1 spec file to load.
|
|
1046
1150
|
tenant (str, Optional): The ESI tenant to use (default: "tranquility").
|
|
1151
|
+
operations (list[str], Optional*): List of operations to filter the spec down.
|
|
1152
|
+
tags (list[str], Optional*): List of tags to filter the spec down.
|
|
1047
1153
|
Functions:
|
|
1048
1154
|
client(): ESIClient
|
|
1049
1155
|
client_async(): ESIClientAsync
|
|
@@ -1065,9 +1171,9 @@ class ESIClientProvider:
|
|
|
1065
1171
|
**kwargs
|
|
1066
1172
|
) -> None:
|
|
1067
1173
|
if type(compatibility_date) is dt.date:
|
|
1068
|
-
self._compatibility_date = self._date_to_string(compatibility_date)
|
|
1174
|
+
self._compatibility_date: str = self._date_to_string(compatibility_date)
|
|
1069
1175
|
else:
|
|
1070
|
-
self._compatibility_date = compatibility_date
|
|
1176
|
+
self._compatibility_date: str = str(compatibility_date)
|
|
1071
1177
|
self._ua_appname = ua_appname
|
|
1072
1178
|
self._ua_version = ua_version
|
|
1073
1179
|
self._ua_url = ua_url
|
|
@@ -1094,7 +1200,7 @@ class ESIClientProvider:
|
|
|
1094
1200
|
return self._client
|
|
1095
1201
|
|
|
1096
1202
|
@property
|
|
1097
|
-
async def client_async(self) -> ESIClientAsync:
|
|
1203
|
+
async def client_async(self) -> ESIClientAsync: # pragma: no cover
|
|
1098
1204
|
if self._client_async is None:
|
|
1099
1205
|
api = await esi_client_factory_async(
|
|
1100
1206
|
compatibility_date=self._compatibility_date,
|
|
@@ -1103,6 +1209,8 @@ class ESIClientProvider:
|
|
|
1103
1209
|
ua_url=self._ua_url,
|
|
1104
1210
|
spec_file=self._spec_file,
|
|
1105
1211
|
tenant=self._tenant,
|
|
1212
|
+
operations=self._operations,
|
|
1213
|
+
tags=self._tags,
|
|
1106
1214
|
**self._kwargs)
|
|
1107
1215
|
self._client_async = ESIClientAsync(api)
|
|
1108
1216
|
return self._client_async
|
esi/rate_limiting.py
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import time
|
|
2
3
|
from django.core.cache import cache
|
|
3
4
|
from esi.exceptions import ESIBucketLimitException
|
|
4
5
|
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
seconds_per_unit = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def interval_to_seconds(s):
|
|
12
|
+
return int(s[:-1]) * seconds_per_unit[s[-1]]
|
|
13
|
+
|
|
5
14
|
|
|
6
15
|
class ESIRateLimitBucket:
|
|
7
16
|
MARKET_DATA_HISTORY = ("market_data_history", 300, 60)
|
|
@@ -16,6 +25,9 @@ class ESIRateLimitBucket:
|
|
|
16
25
|
def choices(cls):
|
|
17
26
|
return [(bucket.slug, bucket.slug.replace("_", " ").title()) for bucket in cls]
|
|
18
27
|
|
|
28
|
+
def __str__(self):
|
|
29
|
+
return f"Rate Limit: {self.slug} - {self.limit} in {self.window}Seconds"
|
|
30
|
+
|
|
19
31
|
|
|
20
32
|
class ESIRateLimiter:
|
|
21
33
|
def __init__(self) -> None:
|
|
@@ -35,44 +47,61 @@ class ESIRateLimiter:
|
|
|
35
47
|
|
|
36
48
|
def get_bucket(self, bucket: ESIRateLimitBucket) -> int:
|
|
37
49
|
# get the value from the bucket
|
|
38
|
-
return
|
|
39
|
-
|
|
40
|
-
|
|
50
|
+
return int(
|
|
51
|
+
cache.get(
|
|
52
|
+
self._slug_to_key(bucket.slug),
|
|
53
|
+
1 # When not found return 1
|
|
54
|
+
)
|
|
41
55
|
)
|
|
42
56
|
|
|
43
|
-
def get_timeout(self, bucket: ESIRateLimitBucket
|
|
44
|
-
|
|
45
|
-
if
|
|
57
|
+
def get_timeout(self, bucket: ESIRateLimitBucket) -> int:
|
|
58
|
+
current_bucket = self.get_bucket(bucket)
|
|
59
|
+
if current_bucket <= 0:
|
|
46
60
|
timeout = cache.ttl(self._slug_to_key(bucket.slug)) + 1
|
|
47
61
|
msg = (
|
|
48
62
|
f"Rate limit for bucket '{bucket.slug}' exceeded: "
|
|
49
|
-
f"{
|
|
63
|
+
f"{current_bucket}/{bucket.limit} in last {bucket.window}s. "
|
|
50
64
|
f"Wait {timeout}s."
|
|
51
65
|
)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
else:
|
|
55
|
-
return timeout # return the time left till reset
|
|
66
|
+
logger.warning(msg)
|
|
67
|
+
return timeout # return the time left till reset
|
|
56
68
|
else:
|
|
57
69
|
return 0 # we are good.
|
|
58
70
|
|
|
59
|
-
def decr_bucket(self, bucket: ESIRateLimitBucket) -> int:
|
|
60
|
-
# decrease the bucket value by
|
|
71
|
+
def decr_bucket(self, bucket: ESIRateLimitBucket, delta: int = 1) -> int:
|
|
72
|
+
# decrease the bucket value by <delta> from the bucket
|
|
61
73
|
return cache.decr(
|
|
62
|
-
self._slug_to_key(bucket.slug)
|
|
74
|
+
self._slug_to_key(bucket.slug),
|
|
75
|
+
delta
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def set_bucket(self, bucket: ESIRateLimitBucket, new_limit: int = 1) -> int:
|
|
79
|
+
# decrease the bucket value by <delta> from the bucket
|
|
80
|
+
return cache.set(
|
|
81
|
+
self._slug_to_key(bucket.slug),
|
|
82
|
+
int(new_limit),
|
|
83
|
+
timeout=bucket.window
|
|
63
84
|
)
|
|
64
85
|
|
|
65
|
-
def check_bucket(self, bucket: ESIRateLimitBucket
|
|
66
|
-
|
|
86
|
+
def check_bucket(self, bucket: ESIRateLimitBucket):
|
|
87
|
+
self.init_bucket(bucket)
|
|
67
88
|
# get the value
|
|
68
|
-
bucket_val =
|
|
89
|
+
bucket_val = self.get_bucket(bucket)
|
|
69
90
|
if bucket_val <= 0:
|
|
70
|
-
timeout =
|
|
91
|
+
timeout = self.get_timeout(bucket)
|
|
71
92
|
if timeout > 0:
|
|
72
|
-
|
|
93
|
+
raise ESIBucketLimitException(bucket, timeout)
|
|
73
94
|
return
|
|
74
|
-
|
|
75
|
-
|
|
95
|
+
|
|
96
|
+
def check_decr_bucket(self, bucket: ESIRateLimitBucket, raise_on_limit: bool = True):
|
|
97
|
+
try:
|
|
98
|
+
self.check_bucket(bucket)
|
|
99
|
+
self.decr_bucket(bucket)
|
|
100
|
+
except ESIBucketLimitException as ex:
|
|
101
|
+
if raise_on_limit:
|
|
102
|
+
raise ex
|
|
103
|
+
else:
|
|
104
|
+
time.sleep(ex.reset)
|
|
76
105
|
|
|
77
106
|
|
|
78
107
|
ESIRateLimits = ESIRateLimiter()
|
esi/signals.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from django.dispatch import Signal
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Basic signal sent after an ESI requests
|
|
5
|
+
from django.dispatch import receiver
|
|
6
|
+
from esi.signals import esi_request_statistics
|
|
7
|
+
|
|
8
|
+
@receiver(esi_request_statistics)
|
|
9
|
+
def esi_callback(sender, operation, status_code, headers, latency, bucket, **kwargs):
|
|
10
|
+
# do stuff
|
|
11
|
+
pass
|
|
12
|
+
"""
|
|
13
|
+
esi_request_statistics = Signal(
|
|
14
|
+
# providing_args=[
|
|
15
|
+
# "operation",
|
|
16
|
+
# "status_code",
|
|
17
|
+
# "headers",
|
|
18
|
+
# "latency",
|
|
19
|
+
# "bucket"
|
|
20
|
+
# ]
|
|
21
|
+
)
|
esi/tests/__init__.py
CHANGED
|
@@ -84,6 +84,32 @@ class SocketAccessError(Exception):
|
|
|
84
84
|
"""Error raised when a test script accesses the network"""
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
class GuardedSocket(socket.socket):
|
|
88
|
+
"""A socket subclass that only allows loopback/localhost."""
|
|
89
|
+
|
|
90
|
+
def _address_is_loopback(self, address):
|
|
91
|
+
|
|
92
|
+
host = None
|
|
93
|
+
if isinstance(address, tuple):
|
|
94
|
+
host = address[0]
|
|
95
|
+
else:
|
|
96
|
+
host = address
|
|
97
|
+
|
|
98
|
+
if isinstance(host, bytes):
|
|
99
|
+
host = host.decode()
|
|
100
|
+
|
|
101
|
+
# quick allow by obvious names
|
|
102
|
+
if host in ("localhost", "127.0.0.1", "::1"):
|
|
103
|
+
return True
|
|
104
|
+
else:
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
def connect(self, address):
|
|
108
|
+
if not self._address_is_loopback(address):
|
|
109
|
+
raise SocketAccessError(f"Attempt to connect to non-localhost address: {address!r}")
|
|
110
|
+
return super().connect(address)
|
|
111
|
+
|
|
112
|
+
|
|
87
113
|
class NoSocketsTestCase(TestCase):
|
|
88
114
|
"""Variation of Django's TestCase class that prevents any network use.
|
|
89
115
|
|
|
@@ -98,15 +124,11 @@ class NoSocketsTestCase(TestCase):
|
|
|
98
124
|
"""
|
|
99
125
|
@classmethod
|
|
100
126
|
def setUpClass(cls):
|
|
101
|
-
cls.
|
|
102
|
-
socket.socket =
|
|
127
|
+
cls._socket_original = socket.socket
|
|
128
|
+
socket.socket = GuardedSocket
|
|
103
129
|
return super().setUpClass()
|
|
104
130
|
|
|
105
131
|
@classmethod
|
|
106
132
|
def tearDownClass(cls):
|
|
107
|
-
socket.socket = cls.
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
@staticmethod
|
|
111
|
-
def guard(*args, **kwargs):
|
|
112
|
-
raise SocketAccessError("Attempted to access network")
|
|
133
|
+
socket.socket = cls._socket_original
|
|
134
|
+
super().tearDownClass()
|
esi/tests/test_decorators.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
"""unit tests for esi decorators"""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
from time import time
|
|
4
5
|
from unittest.mock import patch, Mock
|
|
5
6
|
|
|
7
|
+
from django.core.cache import cache
|
|
6
8
|
from django.contrib.auth.models import User
|
|
7
9
|
from django.contrib.auth.views import redirect_to_login
|
|
8
10
|
from django.contrib.sessions.middleware import SessionMiddleware
|
|
@@ -10,8 +12,10 @@ from django.http import HttpResponse
|
|
|
10
12
|
from django.test import TestCase, RequestFactory
|
|
11
13
|
|
|
12
14
|
from . import _generate_token, _store_as_Token
|
|
15
|
+
from ..rate_limiting import ESIRateLimitBucket, ESIRateLimits
|
|
16
|
+
from ..exceptions import ESIBucketLimitException
|
|
13
17
|
from ..decorators import (
|
|
14
|
-
_check_callback, tokens_required, token_required, single_use_token
|
|
18
|
+
_check_callback, esi_rate_limiter_bucketed, tokens_required, token_required, single_use_token, wait_for_esi_errorlimit_reset
|
|
15
19
|
)
|
|
16
20
|
from ..models import Token, CallbackRedirect
|
|
17
21
|
|
|
@@ -516,3 +520,59 @@ class TestSingleUseTokenRequired(TestCase):
|
|
|
516
520
|
response,
|
|
517
521
|
self.token
|
|
518
522
|
)
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
class TestESIRateLimitDecorator(TestCase):
|
|
526
|
+
|
|
527
|
+
def setUp(self):
|
|
528
|
+
self.bucket = ESIRateLimitBucket(
|
|
529
|
+
"test-bucket",
|
|
530
|
+
1,
|
|
531
|
+
5
|
|
532
|
+
)
|
|
533
|
+
cache.clear()
|
|
534
|
+
|
|
535
|
+
def test_raise(self):
|
|
536
|
+
@esi_rate_limiter_bucketed(bucket=self.bucket)
|
|
537
|
+
def my_func():
|
|
538
|
+
return "Pass"
|
|
539
|
+
|
|
540
|
+
_t = time()
|
|
541
|
+
my_func()
|
|
542
|
+
self.assertLess(time() - _t, 1)
|
|
543
|
+
_t = time()
|
|
544
|
+
with self.assertRaises(ESIBucketLimitException):
|
|
545
|
+
my_func()
|
|
546
|
+
|
|
547
|
+
def test_sleep(self):
|
|
548
|
+
@esi_rate_limiter_bucketed(bucket=self.bucket, raise_on_limit=False)
|
|
549
|
+
def my_func():
|
|
550
|
+
return "Pass"
|
|
551
|
+
|
|
552
|
+
_t = time()
|
|
553
|
+
my_func()
|
|
554
|
+
self.assertLess(time() - _t, 1)
|
|
555
|
+
_t = time()
|
|
556
|
+
my_func()
|
|
557
|
+
duration = time() - _t
|
|
558
|
+
self.assertGreater(duration, 5)
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
class TestESIErrorLimitDecorator(TestCase):
|
|
562
|
+
|
|
563
|
+
def setUp(self):
|
|
564
|
+
cache.clear()
|
|
565
|
+
|
|
566
|
+
def test_sleep(self):
|
|
567
|
+
@wait_for_esi_errorlimit_reset()
|
|
568
|
+
def my_func():
|
|
569
|
+
return "Pass"
|
|
570
|
+
|
|
571
|
+
_t = time()
|
|
572
|
+
my_func()
|
|
573
|
+
self.assertLess(time() - _t, 1)
|
|
574
|
+
cache.set("esi_error_limit_reset", 5, timeout=5)
|
|
575
|
+
_t = time()
|
|
576
|
+
my_func()
|
|
577
|
+
duration = time() - _t
|
|
578
|
+
self.assertGreater(duration, 5)
|