django-esi 8.0.0a2__py3-none-any.whl → 8.0.0a4__py3-none-any.whl

esi/openapi_clients.py

@@ -1,11 +1,14 @@
 import logging
+import pathlib
 import warnings
-from datetime import datetime
+from datetime import datetime, date
 from hashlib import md5
 from typing import Any
 
-from aiopenapi3 import OpenAPI
+from aiopenapi3 import OpenAPI, FileSystemLoader
 from aiopenapi3._types import ResponseDataType, ResponseHeadersType
+from aiopenapi3.errors import HTTPServerError as base_HTTPServerError
+from aiopenapi3.errors import HTTPClientError as base_HTTPClientError
 from aiopenapi3.request import OperationIndex, RequestBase
 from httpx import (
     AsyncClient, Client, HTTPStatusError, RequestError, Response, Timeout,
@@ -16,9 +19,11 @@ from tenacity import (
 )
 
 from django.core.cache import cache
+from django.utils.text import slugify
 
 from esi import app_settings
-from esi.aiopenapi3.plugins import PatchCompatibilityDatePlugin, Trim204ContentType
+from esi.exceptions import HTTPClientError, HTTPServerError, HTTPNotModified
+from esi.aiopenapi3.plugins import Add304ContentType, DjangoESIInit, PatchCompatibilityDatePlugin, Trim204ContentType
 from esi.exceptions import ESIErrorLimitException
 from esi.models import Token
 from esi.stubs import ESIClientStub
@@ -27,6 +32,8 @@ from . import __title__, __url__, __version__
 
 logger = logging.getLogger(__name__)
 
+ETAG_EXPIRY = 60*60*24*7  # 7 days
+
 
 def _time_to_expiry(expires_header: str) -> int:
     """Calculate cache TTL from Expires header
@@ -78,9 +85,19 @@ async def http_retry_async() -> AsyncRetrying:
     )
 
 
+def _load_plugins(app_name):
+    """Load the plugins to make ESI work with this lib.
+
+    Args:
+        app_name (str): app name to use for internal etags
+    """
+    return [PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType(), DjangoESIInit(app_name)]
+
+
 def _load_aiopenapi_client_sync(
         spec_url: str,
         compatibility_date: str,
+        app_name: str,
         user_agent: str,
         tenant: str,
         spec_file: str | None = None) -> OpenAPI:
@@ -89,6 +106,7 @@ def _load_aiopenapi_client_sync(
     Args:
         spec_url (str): _description_
         compatibility_date (str): _description_
+        app_name (str): _description_
         user_agent (str): _description_
         tenant (str): _description_
         spec_file (str | None, optional): _description_. Defaults to None.
@@ -120,21 +138,23 @@ def _load_aiopenapi_client_sync(
             url=spec_url,
             path=spec_file,
             session_factory=session_factory,
+            loader=FileSystemLoader(pathlib.Path(spec_file)),
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType()]
+            plugins=_load_plugins(app_name)
         )
     else:
         return OpenAPI.load_sync(
             url=spec_url,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType()]
+            plugins=_load_plugins(app_name)
         )
 
 
 async def _load_aiopenapi_client_async(
         spec_url: str,
         compatibility_date: str,
+        app_name: str,
         user_agent: str,
         tenant: str,
         spec_file: str | None = None) -> OpenAPI:
@@ -176,18 +196,18 @@ async def _load_aiopenapi_client_async(
             path=spec_file,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType()]
+            plugins=_load_plugins(app_name)
         )
     else:
         return await OpenAPI.load_async(
             url=spec_url,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType()]
+            plugins=_load_plugins(app_name)
         )
 
 
-def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None) -> str:
+def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None = None) -> str:
     """
     AppName/1.2.3 (foo@example.com; +https://gitlab.com/) Django-ESI/1.2.3 (+https://gitlab.com/allianceauth/django-esi)
     Contact Email will be inserted from app_settings.
@@ -228,7 +248,7 @@ def esi_client_factory_sync(
     """
     user_agent = _build_user_agent(ua_appname, ua_version, ua_url)
     spec_url = _get_spec_url()
-    return _load_aiopenapi_client_sync(spec_url, compatibility_date, user_agent, tenant, spec_file)
+    return _load_aiopenapi_client_sync(spec_url, compatibility_date, ua_appname, user_agent, tenant, spec_file)
 
 
 async def esi_client_factory_async(
@@ -250,11 +270,11 @@ async def esi_client_factory_async(
     """
     user_agent = _build_user_agent(ua_appname, ua_version, ua_url)
     spec_url = _get_spec_url()
-    return await _load_aiopenapi_client_async(spec_url, compatibility_date, user_agent, tenant, spec_file)
+    return await _load_aiopenapi_client_async(spec_url, compatibility_date, ua_appname, user_agent, tenant, spec_file)
 
 
 class BaseEsiOperation():
-    def __init__(self, operation, api) -> None:
+    def __init__(self, operation, api: OpenAPI) -> None:
         self.method, self.url, self.operation, self.extra = operation
         self.api = api
         self.token: Token | None = None
@@ -317,12 +337,25 @@ class BaseEsiOperation():
 
         return normalized
 
+    def _etag_key(self) -> str:
+        """Generate a key name used to cache etag responses based on app_name and cache_key
+        Returns:
+            str: Key
+        """
+        # ignore the token this will break the cache
+        return f"{slugify(self.api.app_name)}_etag_{self._cache_key()}"
+
     def _cache_key(self) -> str:
         """Generate a key name used to cache responses based on method, url, args, kwargs
         Returns:
-            str: Key name
+            str: Key
         """
-        data = (self.method + self.url + str(self._args) + str(self._kwargs)).encode('utf-8')
+        # ignore the token this will break the cache
+        ignore_keys = [
+            "token",
+        ]
+        _kwargs = {key: value for key, value in self._kwargs.items() if key not in ignore_keys}
+        data = (self.method + self.url + str(self._args) + str(_kwargs)).encode('utf-8')
         str_hash = md5(data).hexdigest()  # nosec B303
         return f'esi_{str_hash}'
 
@@ -360,7 +393,7 @@ class BaseEsiOperation():
         """
         return any(p.name == "before" or p.name == "after" for p in self.operation.parameters)
 
-    def _get_cache(self, cache_key: str) -> tuple[ResponseHeadersType | None, Any, Response | None]:
+    def _get_cache(self, cache_key: str, etag: str) -> tuple[ResponseHeadersType | None, Any, Response | None]:
         """Retrieve cached response and validate expiry
         Args:
             cache_key (str): The cache key to retrieve
@@ -368,6 +401,9 @@ class BaseEsiOperation():
             tuple[ResponseHeadersType | None, Any, Response | None]: The cached response,
             or None if not found or expired
         """
+        if not app_settings.ESI_CACHE_RESPONSE:
+            return None, None, None
+
         try:
             cached_response = cache.get(cache_key)
         except Exception as e:
@@ -376,18 +412,46 @@ class BaseEsiOperation():
 
         if cached_response:
             logger.debug(f"Cache Hit {self.url}")
-            headers, data = self.parse_cached_request(cached_response)
+            expiry = _time_to_expiry(str(cached_response.headers.get('Expires')))
 
-            expiry = _time_to_expiry(str(headers.get('Expires')))
+            # force check to ensure cache isn't expired
             if expiry < 0:
                 logger.warning("Cache expired by %d seconds, forcing expiry", expiry)
                 return None, None, None
+
+            # check if etag is same before building models from cache
+            if etag:
+                if cached_response.headers.get('Expires') == etag:
+                    # refresh/store the etag's TTL
+                    self._store_etag(cached_response.headers)
+                    raise HTTPNotModified(
+                        status_code=304,
+                        headers=cached_response.headers
+                    )
+
+            # build models
+            headers, data = self.parse_cached_request(cached_response)
             return headers, data, cached_response
 
         return None, None, None
 
+    def _store_etag(self, headers: dict):
+        """
+            Store response etag in cache for 7 days
+        """
+        if "ETag" in headers:
+            cache.set(self._etag_key(), headers["ETag"], timeout=ETAG_EXPIRY)
+
+    def _clear_etag(self):
+        """ Delete the cached etag for this operation.
+        """
+        try:
+            cache.delete(self._etag_key())
+        except Exception as e:
+            logger.error(f"Failed to delete etag {e}", exc_info=True)
+
     def _store_cache(self, cache_key: str, response) -> None:
-        """ Store the response in cache with ETag and TTL.
+        """ Store the response in cache for expiry TTL.
         Args:
             cache_key (str): The cache key to store the response under
             response (Response): The response object to cache
@@ -395,9 +459,6 @@ class BaseEsiOperation():
         if not app_settings.ESI_CACHE_RESPONSE:
             return
 
-        if "ETag" in response.headers:
-            cache.set(f"{cache_key}_etag", response.headers["ETag"])
-
         expires = response.headers.get("Expires")
         ttl = _time_to_expiry(expires) if expires else 0
         if ttl > 0:
@@ -406,6 +467,14 @@ class BaseEsiOperation():
             except Exception as e:
                 logger.error(f"Failed to cache {e}", exc_info=True)
 
+    def _clear_cache(self):
+        """ Delete the cached data for this operation.
+        """
+        try:
+            cache.delete(self._cache_key())
+        except Exception as e:
+            logger.error(f"Failed to delete cache {e}", exc_info=True)
+
     def _validate_token_scopes(self, token: Token) -> None:
         """Validate that the token provided has the required scopes for this ESI operation.
         """
@@ -465,8 +534,9 @@ class EsiOperation(BaseEsiOperation):
 
     def result(
             self,
-            etag: str | None = None,
+            use_etag: bool = True,
             return_response: bool = False,
+            force_refresh: bool = False,
             use_cache: bool = True,
             **extra) -> tuple[Any, Response] | Any:
         """Executes the request and returns the response from ESI for the current operation.
@@ -480,12 +550,17 @@ class EsiOperation(BaseEsiOperation):
         self.body = self._extract_body_param()
         parameters = self._kwargs | extra
         cache_key = self._cache_key()
-        etag_key = f"{cache_key}_etag"
+        etag_key = self._etag_key()
+        etag = None
 
-        if not etag and app_settings.ESI_CACHE_RESPONSE:
+        if force_refresh:
+            self._clear_cache()
+            self._clear_etag()
+
+        if use_etag:
             etag = cache.get(etag_key)
 
-        headers, data, response = self._get_cache(cache_key)
+        headers, data, response = self._get_cache(cache_key, etag=etag) if use_cache else (None, None, None)
 
         if response and use_cache:
             expiry = _time_to_expiry(str(headers.get('Expires')))
@@ -499,24 +574,47 @@ class EsiOperation(BaseEsiOperation):
 
         if not response:
             logger.debug(f"Cache Miss {self.url}")
-            headers, data, response = self._make_request(parameters, etag)
-            if response.status_code == 420:
-                reset = response.headers.get("X-RateLimit-Reset", None)
-                cache.set("esi_error_limit_reset", reset, timeout=reset)
-                raise ESIErrorLimitException(reset=reset)
-            # if response.status_code == 304 and app_settings.ESI_CACHE_RESPONSE:
-            #     cached = cache.get(cache_key)
-            #     if cached:
-            #         return (cached, response) if return_response else cached
-            # we dont want to do this, if we do this we have to store data longer than ttl. rip ram
-            self._store_cache(cache_key, response)
+            try:
+                headers, data, response = self._make_request(parameters, etag)
+                if response.status_code == 420:
+                    reset = response.headers.get("X-RateLimit-Reset", None)
+                    cache.set("esi_error_limit_reset", reset, timeout=reset)
+                    raise ESIErrorLimitException(reset=reset)
+
+            # Shim our exceptions into Django-ESI
+            except base_HTTPServerError as e:
+                raise HTTPServerError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            except base_HTTPClientError as e:
+                raise HTTPClientError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            # Throw a 304 exception for catching.
+            if response.status_code == 304:
+                # refresh/store the etag's TTL
+                self._store_etag(response.headers)
+                raise HTTPNotModified(
+                    status_code=304,
+                    headers=response.headers
+                )
+
+        # last step store cache we dont want to catch the 304 `None` resonses
+        self._store_cache(cache_key, response)
 
         return (data, response) if return_response else data
 
     def results(
             self,
-            etag: str | None = None,
+            use_etag: bool = True,
             return_response: bool = False,
+            force_refresh: bool = False,
             use_cache: bool = True,
             **extra) -> tuple[list[Any], Response | Any | None] | list[Any]:
         all_results: list[Any] = []
@@ -532,7 +630,12 @@ class EsiOperation(BaseEsiOperation):
             total_pages = 1
             while current_page <= total_pages:
                 self._kwargs["page"] = current_page
-                data, response = self.result(etag=etag, return_response=True, **extra)
+                data, response = self.result(
+                    use_etag=use_etag,
+                    return_response=True,
+                    force_refresh=force_refresh,
+                    **extra
+                )
                 last_response = response
                 all_results.extend(data if isinstance(data, list) else [data])
                 total_pages = int(response.headers.get("X-Pages", 1))
@@ -551,7 +654,13 @@ class EsiOperation(BaseEsiOperation):
             else:
                 cursor_param = "after"
             while True:
-                data, response = self.result(etag=etag, return_response=True, use_cache=use_cache, **params)
+                data, response = self.result(
+                    use_etag=use_etag,
+                    return_response=True,
+                    force_refresh=force_refresh,
+                    use_cache=use_cache,
+                    **params
+                )
                 last_response = response
                 if not data:
                     break
@@ -562,7 +671,13 @@ class EsiOperation(BaseEsiOperation):
                 params[cursor_param] = cursor_token
 
         else:
-            data, response = self.result(etag=etag, return_response=True, use_cache=use_cache, **extra)
+            data, response = self.result(
+                use_etag=use_etag,
+                return_response=True,
+                force_refresh=force_refresh,
+                use_cache=use_cache,
+                **extra
+            )
             all_results.extend(data if isinstance(data, list) else [data])
             last_response = response
 
@@ -571,7 +686,8 @@ class EsiOperation(BaseEsiOperation):
     def results_localized(
             self,
             languages: list[str] | str | None = None,
-            **kwargs) -> dict[str, list[Any]]:
+            **kwargs
+    ) -> dict[str, list[Any]]:
         """Executes the request and returns the response from ESI for all default languages and pages (if any).
         Args:
             languages: (list[str], str, optional) language(s) to return instead of default languages
@@ -590,7 +706,7 @@ class EsiOperation(BaseEsiOperation):
                 my_languages.append(lang)
 
         return {
-            language: self.results(language=language, **kwargs)
+            language: self.results(accept_language=language, **kwargs)
             for language in my_languages
         }
 
@@ -664,17 +780,37 @@ class EsiOperationAsync(BaseEsiOperation):
 
         if not response:
             logger.debug(f"Cache Miss {self.url}")
-            headers, data, response = await self._make_request(parameters, etag)
-            if response.status_code == 420:
-                reset = response.headers.get("X-RateLimit-Reset", None)
-                cache.set("esi_error_limit_reset", reset, timeout=reset)
-                raise ESIErrorLimitException(reset=reset)
-            # if response.status_code == 304 and app_settings.ESI_CACHE_RESPONSE:
-            #     cached = cache.get(cache_key)
-            #     if cached:
-            #         return (cached, response) if return_response else cached
-            # we dont want to do this, if we do this we have to store data longer than ttl. rip ram
-            self._store_cache(cache_key, response)
+            try:
+                headers, data, response = await self._make_request(parameters, etag)
+                if response.status_code == 420:
+                    reset = response.headers.get("X-RateLimit-Reset", None)
+                    cache.set("esi_error_limit_reset", reset, timeout=reset)
+                    raise ESIErrorLimitException(reset=reset)
+                self._store_cache(cache_key, response)
+                self._store_etag(response.headers)
+            # Shim our exceptions into Django-ESI
+            except base_HTTPServerError as e:
+                raise HTTPServerError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            except base_HTTPClientError as e:
+                raise HTTPClientError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            # Throw a 304 exception for catching.
+            if response.status_code == 304:
+                # refresh/store the etag's TTL
+                self._store_etag(response.headers)
+                raise HTTPNotModified(
+                    status_code=304,
+                    headers=response.headers
+                )
 
         return (data, response) if return_response else data
 
@@ -732,7 +868,7 @@ class EsiOperationAsync(BaseEsiOperation):
                 my_languages.append(lang)
 
         return {
-            language: self.results(language=language, **extra)
+            language: self.results(accept_language=language, **kwargs)
             for language in my_languages
         }
 
@@ -795,6 +931,7 @@ class ESIClient(ESIClientStub):
     Base ESI Client, provides access to Tags Assets, Characters, etc.
     or Raw aiopenapi3 via sad smiley ._.
     """
+
     def __init__(self, api: OpenAPI) -> None:
         self.api = api
         self._tags = set(api._operationindex._tags.keys())
@@ -804,7 +941,11 @@ class ESIClient(ESIClientStub):
         if tag == "_":
             return self.api._operationindex
 
-        elif tag in set(self.api._operationindex._tags.keys()):
+        # convert pythonic Planetary_Interaction to Planetary Interaction
+        if "_" in tag:
+            tag = tag.replace("_", " ")
+
+        if tag in set(self.api._operationindex._tags.keys()):
             return ESITag(self.api._operationindex._tags[tag], self.api)
 
         raise AttributeError(
@@ -812,12 +953,36 @@ class ESIClient(ESIClientStub):
             f"Available tags: {', '.join(sorted(self._tags))}"
         )
 
+    def purge_all_etags(self):
+        """ Delete all stored etags from the cache for this application
+
+        TODO: consider making this more config agnostic
+        """
+        try:
+            # new lib
+            from django_redis import get_redis_connection
+            _client = get_redis_connection("default")
+        except (NotImplementedError, ModuleNotFoundError):
+            # old lib
+            from django.core.cache import caches
+            default_cache = caches['default']
+            _client = default_cache.get_master_client()
+
+        keys = _client.keys(f":?:{slugify(self.api.app_name)}_etag_*")
+        if keys:
+            deleted = _client.delete(*keys)
+
+        logger.info(f"Deleted {deleted} etag keys")
+
+        return deleted
+
 
 class ESIClientAsync(ESIClientStub):
     """
     Async Base ESI Client, provides access to Tags Assets, Characters, etc.
     or Raw aiopenapi3 via sad smiley ._.
     """
+
     def __init__(self, api: OpenAPI) -> None:
         self.api = api
         self._tags = set(api._operationindex._tags.keys())
@@ -827,7 +992,11 @@ class ESIClientAsync(ESIClientStub):
         if tag == "_":
             return self.api._operationindex
 
-        elif tag in set(self.api._operationindex._tags.keys()):
+        # convert pythonic Planetary_Interaction to Planetary Interaction
+        if "_" in tag:
+            tag = tag.replace("_", " ")
+
+        if tag in set(self.api._operationindex._tags.keys()):
             return ESITagAsync(self.api._operationindex._tags[tag], self.api)
 
         raise AttributeError(
@@ -839,7 +1008,7 @@ class ESIClientAsync(ESIClientStub):
 class ESIClientProvider:
     """Class for providing a single ESI client instance for a whole app
     Args:
-        compatibility_date (str): The compatibility date for the ESI client.
+        compatibility_date (str | date): The compatibility date for the ESI client.
         ua_appname (str): Name of the App for generating a User-Agent,
         ua_version (str): Version of the App for generating a User-Agent,
         ua_url (str, Optional): URL To the Source Code or Documentation for generating a User-Agent,
@@ -855,7 +1024,7 @@ class ESIClientProvider:
 
     def __init__(
         self,
-        compatibility_date: str,
+        compatibility_date: str | date,
         ua_appname: str,
         ua_version: str,
         ua_url: str | None = None,
@@ -863,7 +1032,10 @@ class ESIClientProvider:
         tenant: str = "tranquility",
         **kwargs
     ) -> None:
-        self._compatibility_date = compatibility_date
+        if type(compatibility_date) is date:
+            self._compatibility_date = self._date_to_string(compatibility_date)
+        else:
+            self._compatibility_date = compatibility_date
         self._ua_appname = ua_appname
         self._ua_version = ua_version
         self._ua_url = ua_url
@@ -899,5 +1071,10 @@ class ESIClientProvider:
             self._client_async = ESIClientAsync(api)
         return self._client_async
 
+    @classmethod
+    def _date_to_string(cls, compatibility_date: date) -> str:
+        """Turns a date object in a compatibility_date string"""
+        return f"{compatibility_date.year}-{compatibility_date.month:02}-{compatibility_date.day:02}"
+
     def __str__(self) -> str:
-        return "ESIClientProvider"
+        return f"ESIClientProvider - {self._ua_appname} ({self._ua_version})"