django-esi 8.0.0a3__py3-none-any.whl → 8.0.0b1__py3-none-any.whl

esi/openapi_clients.py

@@ -1,10 +1,11 @@
 import logging
+import pathlib
 import warnings
-from datetime import datetime, date
+import datetime as dt
 from hashlib import md5
 from typing import Any
 
-from aiopenapi3 import OpenAPI
+from aiopenapi3 import OpenAPI, FileSystemLoader
 from aiopenapi3._types import ResponseDataType, ResponseHeadersType
 from aiopenapi3.errors import HTTPServerError as base_HTTPServerError
 from aiopenapi3.errors import HTTPClientError as base_HTTPClientError
@@ -18,18 +19,25 @@ from tenacity import (
 )
 
 from django.core.cache import cache
+from django.utils.text import slugify
 
 from esi import app_settings
 from esi.exceptions import HTTPClientError, HTTPServerError, HTTPNotModified
-from esi.aiopenapi3.plugins import Add304ContentType, PatchCompatibilityDatePlugin, Trim204ContentType
+from esi.aiopenapi3.plugins import (
+    Add304ContentType, DjangoESIInit, PatchCompatibilityDatePlugin,
+    Trim204ContentType, MinifySpec
+)
 from esi.exceptions import ESIErrorLimitException
 from esi.models import Token
 from esi.stubs import ESIClientStub
 
 from . import __title__, __url__, __version__
+from .helpers import pascal_case_string
 
 logger = logging.getLogger(__name__)
 
+ETAG_EXPIRY = 60*60*24*7  # 7 days
+
 
 def _time_to_expiry(expires_header: str) -> int:
     """Calculate cache TTL from Expires header
@@ -39,8 +47,10 @@ def _time_to_expiry(expires_header: str) -> int:
         int: The cache TTL in seconds
     """
     try:
-        expires_dt = datetime.strptime(str(expires_header), '%a, %d %b %Y %H:%M:%S %Z')
-        return max(int((expires_dt - datetime.utcnow()).total_seconds()), 0)
+        expires_dt = dt.datetime.strptime(str(expires_header), '%a, %d %b %Y %H:%M:%S %Z')
+        if expires_dt.tzinfo is None:
+            expires_dt = expires_dt.replace(tzinfo=dt.timezone.utc)
+        return max(int((expires_dt - dt.datetime.now(dt.timezone.utc)).total_seconds()), 0)
     except ValueError:
         return 0
 
@@ -81,17 +91,36 @@ async def http_retry_async() -> AsyncRetrying:
     )
 
 
+def _load_plugins(app_name, tags: list[str]=[], operations: list[str]=[]):
+    """Load the plugins to make ESI work with this lib.
+
+    Args:
+        app_name (str): app name to use for internal etags
+    """
+    return [
+        PatchCompatibilityDatePlugin(),
+        Trim204ContentType(),
+        Add304ContentType(),
+        DjangoESIInit(app_name),
+        MinifySpec(tags, operations)
+    ]
+
+
 def _load_aiopenapi_client_sync(
         spec_url: str,
         compatibility_date: str,
+        app_name: str,
         user_agent: str,
         tenant: str,
-        spec_file: str | None = None) -> OpenAPI:
+        spec_file: str | None = None,
+        tags: list[str] = [],
+        operations: list[str] = []) -> OpenAPI:
     """Create an OpenAPI3 Client from Spec
 
     Args:
         spec_url (str): _description_
         compatibility_date (str): _description_
+        app_name (str): _description_
         user_agent (str): _description_
         tenant (str): _description_
         spec_file (str | None, optional): _description_. Defaults to None.
@@ -123,21 +152,23 @@ def _load_aiopenapi_client_sync(
             url=spec_url,
             path=spec_file,
             session_factory=session_factory,
+            loader=FileSystemLoader(pathlib.Path(spec_file)),
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name, tags, operations)
         )
     else:
         return OpenAPI.load_sync(
             url=spec_url,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name, tags, operations)
         )
 
 
 async def _load_aiopenapi_client_async(
         spec_url: str,
         compatibility_date: str,
+        app_name: str,
         user_agent: str,
         tenant: str,
         spec_file: str | None = None) -> OpenAPI:
@@ -179,18 +210,18 @@ async def _load_aiopenapi_client_async(
             path=spec_file,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name)
         )
     else:
         return await OpenAPI.load_async(
             url=spec_url,
             session_factory=session_factory,
             use_operation_tags=True,
-            plugins=[PatchCompatibilityDatePlugin(), Trim204ContentType(), Add304ContentType()]
+            plugins=_load_plugins(app_name)
         )
 
 
-def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None) -> str:
+def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None = None) -> str:
     """
     AppName/1.2.3 (foo@example.com; +https://gitlab.com/) Django-ESI/1.2.3 (+https://gitlab.com/allianceauth/django-esi)
     Contact Email will be inserted from app_settings.
@@ -201,10 +232,15 @@ def _build_user_agent(ua_appname: str, ua_version: str, ua_url: str | None) -> s
     Returns:
         str: User-Agent string
     """
+
+    # Enforce PascalCase for `ua_appname` and strip whitespace
+    sanitized_ua_appname = pascal_case_string(ua_appname)
+    sanitized_appname = pascal_case_string(__title__)
+
     return (
-        f"{ua_appname}/{ua_version} "
+        f"{sanitized_ua_appname}/{ua_version} "
         f"({app_settings.ESI_USER_CONTACT_EMAIL}{f'; +{ua_url})' if ua_url else ')'} "
-        f"{__title__}/{__version__} (+{__url__})"
+        f"{sanitized_appname}/{__version__} (+{__url__})"
     )
 
 
@@ -217,6 +253,7 @@ def esi_client_factory_sync(
         ua_appname: str, ua_version: str, ua_url: str | None = None,
         spec_file: str | None = None,
         tenant: str = "tranquility",
+        tags: list[str]=[], operations: list[str]=[],
         **kwargs) -> OpenAPI:
     """Generate a new OpenAPI ESI client.
     Args:
@@ -231,7 +268,16 @@ def esi_client_factory_sync(
     """
     user_agent = _build_user_agent(ua_appname, ua_version, ua_url)
     spec_url = _get_spec_url()
-    return _load_aiopenapi_client_sync(spec_url, compatibility_date, user_agent, tenant, spec_file)
+    return _load_aiopenapi_client_sync(
+        spec_url,
+        compatibility_date,
+        ua_appname,
+        user_agent,
+        tenant,
+        spec_file,
+        tags,
+        operations
+    )
 
 
 async def esi_client_factory_async(
@@ -253,11 +299,11 @@ async def esi_client_factory_async(
     """
     user_agent = _build_user_agent(ua_appname, ua_version, ua_url)
     spec_url = _get_spec_url()
-    return await _load_aiopenapi_client_async(spec_url, compatibility_date, user_agent, tenant, spec_file)
+    return await _load_aiopenapi_client_async(spec_url, compatibility_date, ua_appname, user_agent, tenant, spec_file)
 
 
 class BaseEsiOperation():
-    def __init__(self, operation, api) -> None:
+    def __init__(self, operation, api: OpenAPI) -> None:
         self.method, self.url, self.operation, self.extra = operation
         self.api = api
         self.token: Token | None = None
@@ -320,16 +366,24 @@ class BaseEsiOperation():
 
         return normalized
 
+    def _etag_key(self) -> str:
+        """Generate a key name used to cache etag responses based on app_name and cache_key
+        Returns:
+            str: Key
+        """
+        # ignore the token this will break the cache
+        return f"{slugify(self.api.app_name)}_etag_{self._cache_key()}"
+
     def _cache_key(self) -> str:
         """Generate a key name used to cache responses based on method, url, args, kwargs
         Returns:
-            str: Key name
+            str: Key
         """
         # ignore the token this will break the cache
         ignore_keys = [
             "token",
         ]
-        _kwargs = { key:value for key, value in self._kwargs.items() if key not in ignore_keys }
+        _kwargs = {key: value for key, value in self._kwargs.items() if key not in ignore_keys}
         data = (self.method + self.url + str(self._args) + str(_kwargs)).encode('utf-8')
         str_hash = md5(data).hexdigest()  # nosec B303
         return f'esi_{str_hash}'
@@ -376,6 +430,9 @@ class BaseEsiOperation():
             tuple[ResponseHeadersType | None, Any, Response | None]: The cached response,
             or None if not found or expired
         """
+        if not app_settings.ESI_CACHE_RESPONSE:
+            return None, None, None
+
         try:
             cached_response = cache.get(cache_key)
         except Exception as e:
@@ -393,7 +450,9 @@ class BaseEsiOperation():
 
             # check if etag is same before building models from cache
             if etag:
-                if cached_response.headers.get('Expires') == etag:
+                if cached_response.headers.get('ETag') == etag:
+                    # refresh/store the etag's TTL
+                    self._store_etag(cached_response.headers)
                     raise HTTPNotModified(
                         status_code=304,
                         headers=cached_response.headers
@@ -405,8 +464,23 @@ class BaseEsiOperation():
 
         return None, None, None
 
+    def _store_etag(self, headers: dict):
+        """
+            Store response etag in cache for 7 days
+        """
+        if "ETag" in headers:
+            cache.set(self._etag_key(), headers["ETag"], timeout=ETAG_EXPIRY)
+
+    def _clear_etag(self):
+        """ Delete the cached etag for this operation.
+        """
+        try:
+            cache.delete(self._etag_key())
+        except Exception as e:
+            logger.error(f"Failed to delete etag {e}", exc_info=True)
+
     def _store_cache(self, cache_key: str, response) -> None:
-        """ Store the response in cache with ETag and TTL.
+        """ Store the response in cache for expiry TTL.
         Args:
             cache_key (str): The cache key to store the response under
             response (Response): The response object to cache
@@ -414,9 +488,6 @@ class BaseEsiOperation():
         if not app_settings.ESI_CACHE_RESPONSE:
             return
 
-        if "ETag" in response.headers:
-            cache.set(f"{cache_key}_etag", response.headers["ETag"])
-
         expires = response.headers.get("Expires")
         ttl = _time_to_expiry(expires) if expires else 0
         if ttl > 0:
@@ -425,6 +496,14 @@ class BaseEsiOperation():
             except Exception as e:
                 logger.error(f"Failed to cache {e}", exc_info=True)
 
+    def _clear_cache(self):
+        """ Delete the cached data for this operation.
+        """
+        try:
+            cache.delete(self._cache_key())
+        except Exception as e:
+            logger.error(f"Failed to delete cache {e}", exc_info=True)
+
     def _validate_token_scopes(self, token: Token) -> None:
         """Validate that the token provided has the required scopes for this ESI operation.
         """
@@ -484,8 +563,9 @@ class EsiOperation(BaseEsiOperation):
 
     def result(
             self,
-            etag: str | None = None,
+            use_etag: bool = True,
             return_response: bool = False,
+            force_refresh: bool = False,
             use_cache: bool = True,
             **extra) -> tuple[Any, Response] | Any:
         """Executes the request and returns the response from ESI for the current operation.
@@ -499,9 +579,14 @@ class EsiOperation(BaseEsiOperation):
         self.body = self._extract_body_param()
         parameters = self._kwargs | extra
         cache_key = self._cache_key()
-        etag_key = f"{cache_key}_etag"
+        etag_key = self._etag_key()
+        etag = None
 
-        if not etag and app_settings.ESI_CACHE_RESPONSE:
+        if force_refresh:
+            self._clear_cache()
+            self._clear_etag()
+
+        if use_etag:
             etag = cache.get(etag_key)
 
         headers, data, response = self._get_cache(cache_key, etag=etag) if use_cache else (None, None, None)
@@ -524,13 +609,6 @@ class EsiOperation(BaseEsiOperation):
                     reset = response.headers.get("X-RateLimit-Reset", None)
                     cache.set("esi_error_limit_reset", reset, timeout=reset)
                     raise ESIErrorLimitException(reset=reset)
-                self._store_cache(cache_key, response)
-
-                # if response.status_code == 304 and app_settings.ESI_CACHE_RESPONSE:
-                #     cached = cache.get(cache_key)
-                #     if cached:
-                #         return (cached, response) if return_response else cached
-                # we dont want to do this, if we do this we have to store data longer than ttl. rip ram
 
             # Shim our exceptions into Django-ESI
             except base_HTTPServerError as e:
@@ -549,17 +627,24 @@ class EsiOperation(BaseEsiOperation):
 
             # Throw a 304 exception for catching.
             if response.status_code == 304:
+                # refresh/store the etag's TTL
+                self._store_etag(response.headers)
                 raise HTTPNotModified(
                     status_code=304,
                     headers=response.headers
                 )
 
+        # last step store cache we dont want to catch the 304 `None` resonses
+        self._store_cache(cache_key, response)
+        self._store_etag(response.headers)
+
         return (data, response) if return_response else data
 
     def results(
             self,
-            etag: str | None = None,
+            use_etag: bool = True,
             return_response: bool = False,
+            force_refresh: bool = False,
             use_cache: bool = True,
             **extra) -> tuple[list[Any], Response | Any | None] | list[Any]:
         all_results: list[Any] = []
@@ -575,7 +660,12 @@ class EsiOperation(BaseEsiOperation):
             total_pages = 1
             while current_page <= total_pages:
                 self._kwargs["page"] = current_page
-                data, response = self.result(etag=etag, return_response=True, **extra)
+                data, response = self.result(
+                    use_etag=use_etag,
+                    return_response=True,
+                    force_refresh=force_refresh,
+                    **extra
+                )
                 last_response = response
                 all_results.extend(data if isinstance(data, list) else [data])
                 total_pages = int(response.headers.get("X-Pages", 1))
@@ -594,7 +684,13 @@ class EsiOperation(BaseEsiOperation):
             else:
                 cursor_param = "after"
             while True:
-                data, response = self.result(etag=etag, return_response=True, use_cache=use_cache, **params)
+                data, response = self.result(
+                    use_etag=use_etag,
+                    return_response=True,
+                    force_refresh=force_refresh,
+                    use_cache=use_cache,
+                    **params
+                )
                 last_response = response
                 if not data:
                     break
@@ -605,7 +701,13 @@ class EsiOperation(BaseEsiOperation):
                 params[cursor_param] = cursor_token
 
         else:
-            data, response = self.result(etag=etag, return_response=True, use_cache=use_cache, **extra)
+            data, response = self.result(
+                use_etag=use_etag,
+                return_response=True,
+                force_refresh=force_refresh,
+                use_cache=use_cache,
+                **extra
+            )
             all_results.extend(data if isinstance(data, list) else [data])
             last_response = response
 
@@ -614,7 +716,8 @@ class EsiOperation(BaseEsiOperation):
     def results_localized(
             self,
             languages: list[str] | str | None = None,
-            **kwargs) -> dict[str, list[Any]]:
+            **kwargs
+    ) -> dict[str, list[Any]]:
         """Executes the request and returns the response from ESI for all default languages and pages (if any).
         Args:
             languages: (list[str], str, optional) language(s) to return instead of default languages
@@ -707,17 +810,37 @@ class EsiOperationAsync(BaseEsiOperation):
 
         if not response:
             logger.debug(f"Cache Miss {self.url}")
-            headers, data, response = await self._make_request(parameters, etag)
-            if response.status_code == 420:
-                reset = response.headers.get("X-RateLimit-Reset", None)
-                cache.set("esi_error_limit_reset", reset, timeout=reset)
-                raise ESIErrorLimitException(reset=reset)
-            # if response.status_code == 304 and app_settings.ESI_CACHE_RESPONSE:
-            #     cached = cache.get(cache_key)
-            #     if cached:
-            #         return (cached, response) if return_response else cached
-            # we dont want to do this, if we do this we have to store data longer than ttl. rip ram
-            self._store_cache(cache_key, response)
+            try:
+                headers, data, response = await self._make_request(parameters, etag)
+                if response.status_code == 420:
+                    reset = response.headers.get("X-RateLimit-Reset", None)
+                    cache.set("esi_error_limit_reset", reset, timeout=reset)
+                    raise ESIErrorLimitException(reset=reset)
+                self._store_cache(cache_key, response)
+                self._store_etag(response.headers)
+            # Shim our exceptions into Django-ESI
+            except base_HTTPServerError as e:
+                raise HTTPServerError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            except base_HTTPClientError as e:
+                raise HTTPClientError(
+                    status_code=e.status_code,
+                    headers=e.headers,
+                    data=e.data
+                )
+
+            # Throw a 304 exception for catching.
+            if response.status_code == 304:
+                # refresh/store the etag's TTL
+                self._store_etag(response.headers)
+                raise HTTPNotModified(
+                    status_code=304,
+                    headers=response.headers
+                )
 
         return (data, response) if return_response else data
 
@@ -838,6 +961,7 @@ class ESIClient(ESIClientStub):
     Base ESI Client, provides access to Tags Assets, Characters, etc.
     or Raw aiopenapi3 via sad smiley ._.
     """
+
     def __init__(self, api: OpenAPI) -> None:
         self.api = api
         self._tags = set(api._operationindex._tags.keys())
@@ -859,12 +983,36 @@ class ESIClient(ESIClientStub):
             f"Available tags: {', '.join(sorted(self._tags))}"
         )
 
+    def purge_all_etags(self):
+        """ Delete all stored etags from the cache for this application
+
+        TODO: consider making this more config agnostic
+        """
+        try:
+            # new lib
+            from django_redis import get_redis_connection
+            _client = get_redis_connection("default")
+        except (NotImplementedError, ModuleNotFoundError):
+            # old lib
+            from django.core.cache import caches
+            default_cache = caches['default']
+            _client = default_cache.get_master_client()
+
+        keys = _client.keys(f":?:{slugify(self.api.app_name)}_etag_*")
+        if keys:
+            deleted = _client.delete(*keys)
+
+        logger.info(f"Deleted {deleted} etag keys")
+
+        return deleted
+
 
 class ESIClientAsync(ESIClientStub):
     """
     Async Base ESI Client, provides access to Tags Assets, Characters, etc.
     or Raw aiopenapi3 via sad smiley ._.
     """
+
     def __init__(self, api: OpenAPI) -> None:
         self.api = api
         self._tags = set(api._operationindex._tags.keys())
@@ -906,15 +1054,17 @@ class ESIClientProvider:
 
     def __init__(
         self,
-        compatibility_date: str | date,
+        compatibility_date: str | dt.date,
         ua_appname: str,
         ua_version: str,
         ua_url: str | None = None,
         spec_file: None | str = None,
         tenant: str = "tranquility",
+        operations: list[str] = [],
+        tags: list[str] = [],
         **kwargs
     ) -> None:
-        if type(compatibility_date) is date:
+        if type(compatibility_date) is dt.date:
             self._compatibility_date = self._date_to_string(compatibility_date)
         else:
             self._compatibility_date = compatibility_date
@@ -924,6 +1074,8 @@ class ESIClientProvider:
         self._spec_file = spec_file
         self._tenant = tenant
         self._kwargs = kwargs
+        self._operations = operations
+        self._tags = tags
 
     @property
     def client(self) -> ESIClient:
@@ -935,6 +1087,8 @@ class ESIClientProvider:
                 ua_url=self._ua_url,
                 spec_file=self._spec_file,
                 tenant=self._tenant,
+                operations=self._operations,
+                tags=self._tags,
                 **self._kwargs)
             self._client = ESIClient(api)
         return self._client
@@ -954,10 +1108,9 @@ class ESIClientProvider:
         return self._client_async
 
     @classmethod
-    def _date_to_string(cls, compatibility_date: date) -> str:
+    def _date_to_string(cls, compatibility_date: dt.date) -> str:
         """Turns a date object in a compatibility_date string"""
         return f"{compatibility_date.year}-{compatibility_date.month:02}-{compatibility_date.day:02}"
 
-
     def __str__(self) -> str:
-        return "ESIClientProvider"
+        return f"ESIClientProvider - {self._ua_appname} ({self._ua_version})"