django-esi 8.0.0a3__py3-none-any.whl → 8.0.0a4__py3-none-any.whl

esi/openapi_clients.py

@@ -1,10 +1,11 @@
 import logging
+import pathlib
 import warnings
 from datetime import datetime, date
 from hashlib import md5
 from typing import Any
 
-from aiopenapi3 import OpenAPI
+from aiopenapi3 import OpenAPI, FileSystemLoader
 from aiopenapi3._types import ResponseDataType, ResponseHeadersType
 from aiopenapi3.errors import HTTPServerError as base_HTTPServerError
 from aiopenapi3.errors import HTTPClientError as base_HTTPClientError
@@ -18,10 +19,11 @@ from tenacity import (
 )
 
 from django.core.cache import cache
+from django.utils.text import slugify
 
 from esi import app_settings
 from esi.exceptions import HTTPClientError, HTTPServerError, HTTPNotModified
-from esi.aiopenapi3.plugins import Add304ContentType, PatchCompatibilityDatePlugin, Trim204ContentType
+from esi.aiopenapi3.plugins import Add304ContentType, DjangoESIInit, PatchCompatibilityDatePlugin, Trim204ContentType
 from esi.exceptions import ESIErrorLimitException
 from esi.models import Token
 from esi.stubs import ESIClientStub
@@ -30,6 +32,8 @@ from . import __title__, __url__, __version__
 
 logger = logging.getLogger(__name__)
 
+ETAG_EXPIRY = 60*60*24*7  # 7 days
+
 
 def _time_to_expiry(expires_header: str) -> int:
     """Calculate cache TTL from Expires header
@@ -81,9 +85,19 @@ async def http_retry_async() -> AsyncRetrying:
     )
 
 
+def _load_plugins(app_name):
+    """Load the plugins to make ESI work with this lib.
+
+    Args:
+        app_name (str): app name to use for internal etags
+    """
+    return [PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType(), DjangoESIInit(app_name)]
+
+
 def _load_aiopenapi_client_sync(
         spec_url: str,
         compatibility_date: str,
+        app_name: str,
         user_agent: str,
         tenant: str,
         spec_file: str | None = None) -> OpenAPI:
@@ -92,6 +106,7 @@ def _load_aiopenapi_client_sync(
     Args:
         spec_url (str): _description_
         compatibility_date (str): _description_
+        app_name (str): _description_
         user_agent (str): _description_
         tenant (str): _description_
         spec_file (str | None, optional): _description_. Defaults to None.
@@ -123,21 +138,23 @@ def _load_aiopenapi_client_sync(
             url=spec_url,
             path=spec_file,
             session_factory=session_factory,
+            loader=FileSystemLoader(pathlib.Path(spec_file)),
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name)
         )
     else:
         return OpenAPI.load_sync(
             url=spec_url,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name)
         )
 
 
 async def _load_aiopenapi_client_async(
         spec_url: str,
         compatibility_date: str,
+        app_name: str,
         user_agent: str,
         tenant: str,
         spec_file: str | None = None) -> OpenAPI:
@@ -179,18 +196,18 @@ async def _load_aiopenapi_client_async(
             path=spec_file,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name)
         )
     else:
         return await OpenAPI.load_async(
             url=spec_url,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name)
         )
 
 
-def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None) -> str:
+def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None = None) -> str:
     """
     AppName/1.2.3 (foo@example.com; +https://gitlab.com/) Django-ESI/1.2.3 (+https://gitlab.com/allianceauth/django-esi)
     Contact Email will be inserted from app_settings.
@@ -231,7 +248,7 @@ def esi_client_factory_sync(
     """
     user_agent = _build_user_agent(ua_appname, ua_version, ua_url)
     spec_url = _get_spec_url()
-    return _load_aiopenapi_client_sync(spec_url, compatibility_date, user_agent, tenant, spec_file)
+    return _load_aiopenapi_client_sync(spec_url, compatibility_date, ua_appname, user_agent, tenant, spec_file)
 
 
 async def esi_client_factory_async(
@@ -253,11 +270,11 @@ async def esi_client_factory_async(
     """
     user_agent = _build_user_agent(ua_appname, ua_version, ua_url)
     spec_url = _get_spec_url()
-    return await _load_aiopenapi_client_async(spec_url, compatibility_date, user_agent, tenant, spec_file)
+    return await _load_aiopenapi_client_async(spec_url, compatibility_date, ua_appname, user_agent, tenant, spec_file)
 
 
 class BaseEsiOperation():
-    def __init__(self, operation, api) -> None:
+    def __init__(self, operation, api: OpenAPI) -> None:
         self.method, self.url, self.operation, self.extra = operation
         self.api = api
         self.token: Token | None = None
@@ -320,16 +337,24 @@ class BaseEsiOperation():
 
         return normalized
 
+    def _etag_key(self) -> str:
+        """Generate a key name used to cache etag responses based on app_name and cache_key
+        Returns:
+            str: Key
+        """
+        # ignore the token this will break the cache
+        return f"{slugify(self.api.app_name)}_etag_{self._cache_key()}"
+
     def _cache_key(self) -> str:
         """Generate a key name used to cache responses based on method, url, args, kwargs
         Returns:
-            str: Key name
+            str: Key
         """
         # ignore the token this will break the cache
         ignore_keys = [
             "token",
         ]
-        _kwargs = { key:value for key, value in self._kwargs.items() if key not in ignore_keys }
+        _kwargs = {key: value for key, value in self._kwargs.items() if key not in ignore_keys}
         data = (self.method + self.url + str(self._args) + str(_kwargs)).encode('utf-8')
         str_hash = md5(data).hexdigest()  # nosec B303
         return f'esi_{str_hash}'
@@ -376,6 +401,9 @@ class BaseEsiOperation():
             tuple[ResponseHeadersType | None, Any, Response | None]: The cached response,
             or None if not found or expired
         """
+        if not app_settings.ESI_CACHE_RESPONSE:
+            return None, None, None
+
         try:
             cached_response = cache.get(cache_key)
         except Exception as e:
@@ -394,6 +422,8 @@ class BaseEsiOperation():
             # check if etag is same before building models from cache
             if etag:
                 if cached_response.headers.get('Expires') == etag:
+                    # refresh/store the etag's TTL
+                    self._store_etag(cached_response.headers)
                     raise HTTPNotModified(
                         status_code=304,
                         headers=cached_response.headers
@@ -405,8 +435,23 @@ class BaseEsiOperation():
 
         return None, None, None
 
+    def _store_etag(self, headers: dict):
+        """
+            Store response etag in cache for 7 days
+        """
+        if "ETag" in headers:
+            cache.set(self._etag_key(), headers["ETag"], timeout=ETAG_EXPIRY)
+
+    def _clear_etag(self):
+        """ Delete the cached etag for this operation.
+        """
+        try:
+            cache.delete(self._etag_key())
+        except Exception as e:
+            logger.error(f"Failed to delete etag {e}", exc_info=True)
+
     def _store_cache(self, cache_key: str, response) -> None:
-        """ Store the response in cache with ETag and TTL.
+        """ Store the response in cache for expiry TTL.
         Args:
             cache_key (str): The cache key to store the response under
             response (Response): The response object to cache
@@ -414,9 +459,6 @@ class BaseEsiOperation():
         if not app_settings.ESI_CACHE_RESPONSE:
             return
 
-        if "ETag" in response.headers:
-            cache.set(f"{cache_key}_etag", response.headers["ETag"])
-
         expires = response.headers.get("Expires")
         ttl = _time_to_expiry(expires) if expires else 0
         if ttl > 0:
@@ -425,6 +467,14 @@ class BaseEsiOperation():
             except Exception as e:
                 logger.error(f"Failed to cache {e}", exc_info=True)
 
+    def _clear_cache(self):
+        """ Delete the cached data for this operation.
+        """
+        try:
+            cache.delete(self._cache_key())
+        except Exception as e:
+            logger.error(f"Failed to delete cache {e}", exc_info=True)
+
     def _validate_token_scopes(self, token: Token) -> None:
         """Validate that the token provided has the required scopes for this ESI operation.
         """
@@ -484,8 +534,9 @@ class EsiOperation(BaseEsiOperation):
 
     def result(
             self,
-            etag: str | None = None,
+            use_etag: bool = True,
             return_response: bool = False,
+            force_refresh: bool = False,
             use_cache: bool = True,
             **extra) -> tuple[Any, Response] | Any:
         """Executes the request and returns the response from ESI for the current operation.
@@ -499,9 +550,14 @@ class EsiOperation(BaseEsiOperation):
         self.body = self._extract_body_param()
         parameters = self._kwargs | extra
         cache_key = self._cache_key()
-        etag_key = f"{cache_key}_etag"
+        etag_key = self._etag_key()
+        etag = None
 
-        if not etag and app_settings.ESI_CACHE_RESPONSE:
+        if force_refresh:
+            self._clear_cache()
+            self._clear_etag()
+
+        if use_etag:
             etag = cache.get(etag_key)
 
         headers, data, response = self._get_cache(cache_key, etag=etag) if use_cache else (None, None, None)
@@ -524,13 +580,6 @@ class EsiOperation(BaseEsiOperation):
                     reset = response.headers.get("X-RateLimit-Reset", None)
                     cache.set("esi_error_limit_reset", reset, timeout=reset)
                     raise ESIErrorLimitException(reset=reset)
-                self._store_cache(cache_key, response)
-
-                # if response.status_code == 304 and app_settings.ESI_CACHE_RESPONSE:
-                #     cached = cache.get(cache_key)
-                #     if cached:
-                #         return (cached, response) if return_response else cached
-                # we dont want to do this, if we do this we have to store data longer than ttl. rip ram
 
             # Shim our exceptions into Django-ESI
             except base_HTTPServerError as e:
@@ -549,17 +598,23 @@ class EsiOperation(BaseEsiOperation):
 
             # Throw a 304 exception for catching.
             if response.status_code == 304:
+                # refresh/store the etag's TTL
+                self._store_etag(response.headers)
                 raise HTTPNotModified(
                     status_code=304,
                     headers=response.headers
                 )
 
+        # last step store cache we dont want to catch the 304 `None` resonses
+        self._store_cache(cache_key, response)
+
         return (data, response) if return_response else data
 
     def results(
             self,
-            etag: str | None = None,
+            use_etag: bool = True,
             return_response: bool = False,
+            force_refresh: bool = False,
             use_cache: bool = True,
             **extra) -> tuple[list[Any], Response | Any | None] | list[Any]:
         all_results: list[Any] = []
@@ -575,7 +630,12 @@ class EsiOperation(BaseEsiOperation):
             total_pages = 1
             while current_page <= total_pages:
                 self._kwargs["page"] = current_page
-                data, response = self.result(etag=etag, return_response=True, **extra)
+                data, response = self.result(
+                    use_etag=use_etag,
+                    return_response=True,
+                    force_refresh=force_refresh,
+                    **extra
+                )
                 last_response = response
                 all_results.extend(data if isinstance(data, list) else [data])
                 total_pages = int(response.headers.get("X-Pages", 1))
@@ -594,7 +654,13 @@ class EsiOperation(BaseEsiOperation):
             else:
                 cursor_param = "after"
             while True:
-                data, response = self.result(etag=etag, return_response=True, use_cache=use_cache, **params)
+                data, response = self.result(
+                    use_etag=use_etag,
+                    return_response=True,
+                    force_refresh=force_refresh,
+                    use_cache=use_cache,
+                    **params
+                )
                 last_response = response
                 if not data:
                     break
@@ -605,7 +671,13 @@ class EsiOperation(BaseEsiOperation):
                 params[cursor_param] = cursor_token
 
         else:
-            data, response = self.result(etag=etag, return_response=True, use_cache=use_cache, **extra)
+            data, response = self.result(
+                use_etag=use_etag,
+                return_response=True,
+                force_refresh=force_refresh,
+                use_cache=use_cache,
+                **extra
+            )
             all_results.extend(data if isinstance(data, list) else [data])
             last_response = response
 
@@ -614,7 +686,8 @@ class EsiOperation(BaseEsiOperation):
     def results_localized(
             self,
             languages: list[str] | str | None = None,
-            **kwargs) -> dict[str, list[Any]]:
+            **kwargs
+    ) -> dict[str, list[Any]]:
         """Executes the request and returns the response from ESI for all default languages and pages (if any).
         Args:
             languages: (list[str], str, optional) language(s) to return instead of default languages
@@ -707,17 +780,37 @@ class EsiOperationAsync(BaseEsiOperation):
 
         if not response:
             logger.debug(f"Cache Miss {self.url}")
-            headers, data, response = await self._make_request(parameters, etag)
-            if response.status_code == 420:
-                reset = response.headers.get("X-RateLimit-Reset", None)
-                cache.set("esi_error_limit_reset", reset, timeout=reset)
-                raise ESIErrorLimitException(reset=reset)
-            # if response.status_code == 304 and app_settings.ESI_CACHE_RESPONSE:
-            #     cached = cache.get(cache_key)
-            #     if cached:
-            #         return (cached, response) if return_response else cached
-            # we dont want to do this, if we do this we have to store data longer than ttl. rip ram
-            self._store_cache(cache_key, response)
+            try:
+                headers, data, response = await self._make_request(parameters, etag)
+                if response.status_code == 420:
+                    reset = response.headers.get("X-RateLimit-Reset", None)
+                    cache.set("esi_error_limit_reset", reset, timeout=reset)
+                    raise ESIErrorLimitException(reset=reset)
+                self._store_cache(cache_key, response)
+                self._store_etag(response.headers)
+            # Shim our exceptions into Django-ESI
+            except base_HTTPServerError as e:
+                raise HTTPServerError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            except base_HTTPClientError as e:
+                raise HTTPClientError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            # Throw a 304 exception for catching.
+            if response.status_code == 304:
+                # refresh/store the etag's TTL
+                self._store_etag(response.headers)
+                raise HTTPNotModified(
+                    status_code=304,
+                    headers=response.headers
+                )
 
         return (data, response) if return_response else data
 
@@ -838,6 +931,7 @@ class ESIClient(ESIClientStub):
     Base ESI Client, provides access to Tags Assets, Characters, etc.
     or Raw aiopenapi3 via sad smiley ._.
     """
+
     def __init__(self, api: OpenAPI) -> None:
         self.api = api
         self._tags = set(api._operationindex._tags.keys())
@@ -859,12 +953,36 @@ class ESIClient(ESIClientStub):
             f"Available tags: {', '.join(sorted(self._tags))}"
         )
 
+    def purge_all_etags(self):
+        """ Delete all stored etags from the cache for this application
+
+        TODO: consider making this more config agnostic
+        """
+        try:
+            # new lib
+            from django_redis import get_redis_connection
+            _client = get_redis_connection("default")
+        except (NotImplementedError, ModuleNotFoundError):
+            # old lib
+            from django.core.cache import caches
+            default_cache = caches['default']
+            _client = default_cache.get_master_client()
+
+        keys = _client.keys(f":?:{slugify(self.api.app_name)}_etag_*")
+        if keys:
+            deleted = _client.delete(*keys)
+
+        logger.info(f"Deleted {deleted} etag keys")
+
+        return deleted
+
 
 class ESIClientAsync(ESIClientStub):
     """
     Async Base ESI Client, provides access to Tags Assets, Characters, etc.
     or Raw aiopenapi3 via sad smiley ._.
     """
+
     def __init__(self, api: OpenAPI) -> None:
         self.api = api
         self._tags = set(api._operationindex._tags.keys())
@@ -958,6 +1076,5 @@ class ESIClientProvider:
         """Turns a date object in a compatibility_date string"""
         return f"{compatibility_date.year}-{compatibility_date.month:02}-{compatibility_date.day:02}"
 
-
     def __str__(self) -> str:
-        return "ESIClientProvider"
+        return f"ESIClientProvider - {self._ua_appname} ({self._ua_version})"