hishel 0.0.21__tar.gz → 0.0.22__tar.gz

{hishel-0.0.21 → hishel-0.0.22}/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.0.22 (31th January, 2024)
+
+- Make `FileStorage` to check staleness of all cache files with set interval. (#169)
+- Support AWS S3 storages. (#164)
+- Move `typing_extensions` from requirements.txt to pyproject.toml. (#161)
+
 ## 0.0.21 (29th December, 2023)
 
 - Fix inner transport and connection pool instances closing. (#147)

{hishel-0.0.21 → hishel-0.0.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hishel
-Version: 0.0.21
+Version: 0.0.22
 Summary: Persistent cache implementation for httpx and httpcore
 Project-URL: Homepage, https://hishel.com
 Project-URL: Source, https://github.com/karpetrosyan/hishel
@@ -24,8 +24,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Internet :: WWW/HTTP
 Requires-Python: >=3.8
 Requires-Dist: httpx>=0.22.0
+Requires-Dist: typing-extensions>=4.8.0
 Provides-Extra: redis
 Requires-Dist: redis==5.0.1; extra == 'redis'
+Provides-Extra: s3
+Requires-Dist: boto3<=1.15.3,>=1.15.0; extra == 's3'
 Provides-Extra: sqlite
 Requires-Dist: anysqlite>=0.0.5; extra == 'sqlite'
 Provides-Extra: yaml
@@ -70,7 +73,7 @@ Description-Content-Type: text/markdown
 - 🧠 **Smart**: Attempts to clearly implement RFC 9111, understands `Vary`, `Etag`, `Last-Modified`,  `Cache-Control`, and `Expires` headers, and *handles response re-validation automatically*.
 - ⚙️  **Configurable**: You have complete control over how the responses are stored and serialized.
 - 📦 **From the package**:
-    - Built-in support for [File system](https://en.wikipedia.org/wiki/File_system), [Redis](https://en.wikipedia.org/wiki/Redis), and [SQLite](https://en.wikipedia.org/wiki/SQLite) backends.
+    - Built-in support for [File system](https://en.wikipedia.org/wiki/File_system), [Redis](https://en.wikipedia.org/wiki/Redis), [SQLite](https://en.wikipedia.org/wiki/SQLite), and [AWS S3](https://aws.amazon.com/s3/) backends.
     - Built-in support for [JSON](https://en.wikipedia.org/wiki/JSON), [YAML](https://en.wikipedia.org/wiki/YAML), and [pickle](https://docs.python.org/3/library/pickle.html) serializers.
 - 🚀 **Very fast**: Your requests will be even faster if there are *no IO operations*.
 
@@ -153,6 +156,12 @@ Help us grow and continue developing good software for you ❤️
 
 # Changelog
 
+## 0.0.22 (31th January, 2024)
+
+- Make `FileStorage` to check staleness of all cache files with set interval. (#169)
+- Support AWS S3 storages. (#164)
+- Move `typing_extensions` from requirements.txt to pyproject.toml. (#161)
+
 ## 0.0.21 (29th December, 2023)
 
 - Fix inner transport and connection pool instances closing. (#147)

{hishel-0.0.21 → hishel-0.0.22}/README.md

@@ -36,7 +36,7 @@
 - 🧠 **Smart**: Attempts to clearly implement RFC 9111, understands `Vary`, `Etag`, `Last-Modified`,  `Cache-Control`, and `Expires` headers, and *handles response re-validation automatically*.
 - ⚙️  **Configurable**: You have complete control over how the responses are stored and serialized.
 - 📦 **From the package**:
-    - Built-in support for [File system](https://en.wikipedia.org/wiki/File_system), [Redis](https://en.wikipedia.org/wiki/Redis), and [SQLite](https://en.wikipedia.org/wiki/SQLite) backends.
+    - Built-in support for [File system](https://en.wikipedia.org/wiki/File_system), [Redis](https://en.wikipedia.org/wiki/Redis), [SQLite](https://en.wikipedia.org/wiki/SQLite), and [AWS S3](https://aws.amazon.com/s3/) backends.
     - Built-in support for [JSON](https://en.wikipedia.org/wiki/JSON), [YAML](https://en.wikipedia.org/wiki/YAML), and [pickle](https://docs.python.org/3/library/pickle.html) serializers.
 - 🚀 **Very fast**: Your requests will be even faster if there are *no IO operations*.
 

{hishel-0.0.21 → hishel-0.0.22}/hishel/__init__.py

@@ -14,4 +14,4 @@ def install_cache() -> None:  # pragma: no cover
     httpx.Client = CacheClient  # type: ignore
 
 
-__version__ = "0.0.21"
+__version__ = "0.0.22"

{hishel-0.0.21 → hishel-0.0.22}/hishel/_async/_storages.py

@@ -5,6 +5,11 @@ import warnings
 from copy import deepcopy
 from pathlib import Path
 
+try:
+    import boto3
+except ImportError:  # pragma: no cover
+    boto3 = None  # type: ignore
+
 try:
     import anysqlite
 except ImportError:  # pragma: no cover
@@ -16,13 +21,14 @@ from typing_extensions import TypeAlias
 from hishel._serializers import BaseSerializer, clone_model
 
 from .._files import AsyncFileManager
+from .._s3 import AsyncS3Manager
 from .._serializers import JSONSerializer, Metadata
 from .._synchronization import AsyncLock
 from .._utils import float_seconds_to_int_milliseconds
 
 logger = logging.getLogger("hishel.storages")
 
-__all__ = ("AsyncFileStorage", "AsyncRedisStorage", "AsyncSQLiteStorage", "AsyncInMemoryStorage")
+__all__ = ("AsyncFileStorage", "AsyncRedisStorage", "AsyncSQLiteStorage", "AsyncInMemoryStorage", "AsyncS3Storage")
 
 StoredResponse: TypeAlias = tp.Tuple[Response, Request, Metadata]
 
@@ -61,6 +67,9 @@ class AsyncFileStorage(AsyncBaseStorage):
     :type base_path: tp.Optional[Path], optional
     :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
     :type ttl: tp.Optional[tp.Union[int, float]], optional
+    :param check_ttl_every: How often in seconds to check staleness of **all** cache files.
+        Makes sense only with set `ttl`, defaults to 60
+    :type check_ttl_every: tp.Union[int, float]
     """
 
     def __init__(
@@ -68,6 +77,7 @@ class AsyncFileStorage(AsyncBaseStorage):
         serializer: tp.Optional[BaseSerializer] = None,
         base_path: tp.Optional[Path] = None,
         ttl: tp.Optional[tp.Union[int, float]] = None,
+        check_ttl_every: tp.Union[int, float] = 60,
     ) -> None:
         super().__init__(serializer, ttl)
 
@@ -78,6 +88,8 @@ class AsyncFileStorage(AsyncBaseStorage):
 
         self._file_manager = AsyncFileManager(is_binary=self._serializer.is_binary)
         self._lock = AsyncLock()
+        self._check_ttl_every = check_ttl_every
+        self._last_cleaned = time.monotonic()
 
     async def store(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
         """
@@ -99,7 +111,7 @@ class AsyncFileStorage(AsyncBaseStorage):
                 str(response_path),
                 self._serializer.dumps(response=response, request=request, metadata=metadata),
             )
-        await self._remove_expired_caches()
+        await self._remove_expired_caches(response_path)
 
     async def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
         """
@@ -113,7 +125,7 @@ class AsyncFileStorage(AsyncBaseStorage):
 
         response_path = self._base_path / key
 
-        await self._remove_expired_caches()
+        await self._remove_expired_caches(response_path)
         async with self._lock:
             if response_path.exists():
                 return self._serializer.loads(await self._file_manager.read_from(str(response_path)))
@@ -122,10 +134,18 @@ class AsyncFileStorage(AsyncBaseStorage):
     async def aclose(self) -> None:  # pragma: no cover
         return
 
-    async def _remove_expired_caches(self) -> None:
+    async def _remove_expired_caches(self, response_path: Path) -> None:
         if self._ttl is None:
             return
 
+        if time.monotonic() - self._last_cleaned < self._check_ttl_every:
+            if response_path.is_file():
+                age = time.time() - response_path.stat().st_mtime
+                if age > self._ttl:
+                    response_path.unlink()
+            return
+
+        self._last_cleaned = time.monotonic()
         async with self._lock:
             for file in self._base_path.iterdir():
                 if file.is_file():
@@ -402,3 +422,89 @@ class AsyncInMemoryStorage(AsyncBaseStorage):
 
             for key in keys_to_remove:
                 self._cache.remove_key(key)
+
+
+class AsyncS3Storage(AsyncBaseStorage):  # pragma: no cover
+    """
+    AWS S3 storage.
+
+    :param bucket_name: The name of the bucket to store the responses in
+    :type bucket_name: str
+    :param serializer: Serializer capable of serializing and de-serializing http responses, defaults to None
+    :type serializer: tp.Optional[BaseSerializer], optional
+    :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
+    :type ttl: tp.Optional[tp.Union[int, float]], optional
+    :param client: A client for S3, defaults to None
+    :type client: tp.Optional[tp.Any], optional
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        serializer: tp.Optional[BaseSerializer] = None,
+        ttl: tp.Optional[tp.Union[int, float]] = None,
+        client: tp.Optional[tp.Any] = None,
+    ) -> None:
+        super().__init__(serializer, ttl)
+
+        if boto3 is None:  # pragma: no cover
+            raise RuntimeError(
+                (
+                    f"The `{type(self).__name__}` was used, but the required packages were not found. "
+                    "Check that you have `Hishel` installed with the `s3` extension as shown.\n"
+                    "```pip install hishel[s3]```"
+                )
+            )
+
+        self._bucket_name = bucket_name
+        client = client or boto3.client("s3")
+        self._s3_manager = AsyncS3Manager(client=client, bucket_name=bucket_name, is_binary=self._serializer.is_binary)
+        self._lock = AsyncLock()
+
+    async def store(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
+        """
+        Stores the response in the cache.
+
+        :param key: Hashed value of concatenated HTTP method and URI
+        :type key: str
+        :param response: An HTTP response
+        :type response: httpcore.Response
+        :param request: An HTTP request
+        :type request: httpcore.Request
+        :param metadata: Additioal information about the stored response
+        :type metadata: Metadata`
+        """
+
+        async with self._lock:
+            serialized = self._serializer.dumps(response=response, request=request, metadata=metadata)
+            await self._s3_manager.write_to(path=key, data=serialized)
+
+        await self._remove_expired_caches()
+
+    async def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
+        """
+        Retreives the response from the cache using his key.
+
+        :param key: Hashed value of concatenated HTTP method and URI
+        :type key: str
+        :return: An HTTP response and its HTTP request.
+        :rtype: tp.Optional[StoredResponse]
+        """
+
+        await self._remove_expired_caches()
+        async with self._lock:
+            try:
+                return self._serializer.loads(await self._s3_manager.read_from(path=key))
+            except Exception:
+                return None
+
+    async def aclose(self) -> None:  # pragma: no cover
+        return
+
+    async def _remove_expired_caches(self) -> None:
+        if self._ttl is None:
+            return
+
+        async with self._lock:
+            converted_ttl = float_seconds_to_int_milliseconds(self._ttl)
+            await self._s3_manager.remove_expired(ttl=converted_ttl)

hishel-0.0.22/hishel/_s3.py

@@ -0,0 +1,54 @@
+import typing as tp
+from datetime import datetime, timedelta, timezone
+
+from anyio import to_thread
+
+
+class S3Manager:
+    def __init__(self, client: tp.Any, bucket_name: str, is_binary: bool = False):
+        self._client = client
+        self._bucket_name = bucket_name
+        self._is_binary = is_binary
+
+    def write_to(self, path: str, data: tp.Union[bytes, str]) -> None:
+        path = "hishel-" + path
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+
+        self._client.put_object(Bucket=self._bucket_name, Key=path, Body=data)
+
+    def read_from(self, path: str) -> tp.Union[bytes, str]:
+        path = "hishel-" + path
+        response = self._client.get_object(
+            Bucket=self._bucket_name,
+            Key=path,
+        )
+
+        content = response["Body"].read()
+
+        if self._is_binary:  # pragma: no cover
+            return tp.cast(bytes, content)
+
+        return tp.cast(str, content.decode("utf-8"))
+
+    def remove_expired(self, ttl: int) -> None:
+        for obj in self._client.list_objects(Bucket=self._bucket_name).get("Contents", []):
+            if not obj["Key"].startswith("hishel-"):  # pragma: no cover
+                continue
+
+            if datetime.now(timezone.utc) - obj["LastModified"] > timedelta(milliseconds=ttl):
+                self._client.delete_object(Bucket=self._bucket_name, Key=obj["Key"])
+
+
+class AsyncS3Manager:
+    def __init__(self, client: tp.Any, bucket_name: str, is_binary: bool = False):
+        self._sync_manager = S3Manager(client, bucket_name, is_binary)
+
+    async def write_to(self, path: str, data: tp.Union[bytes, str]) -> None:
+        return await to_thread.run_sync(self._sync_manager.write_to, path, data)
+
+    async def read_from(self, path: str) -> tp.Union[bytes, str]:
+        return await to_thread.run_sync(self._sync_manager.read_from, path)
+
+    async def remove_expired(self, ttl: int) -> None:
+        return await to_thread.run_sync(self._sync_manager.remove_expired, ttl)

{hishel-0.0.21 → hishel-0.0.22}/hishel/_sync/_storages.py

@@ -5,6 +5,11 @@ import warnings
 from copy import deepcopy
 from pathlib import Path
 
+try:
+    import boto3
+except ImportError:  # pragma: no cover
+    boto3 = None  # type: ignore
+
 try:
     import sqlite3
 except ImportError:  # pragma: no cover
@@ -16,13 +21,14 @@ from typing_extensions import TypeAlias
 from hishel._serializers import BaseSerializer, clone_model
 
 from .._files import FileManager
+from .._s3 import S3Manager
 from .._serializers import JSONSerializer, Metadata
 from .._synchronization import Lock
 from .._utils import float_seconds_to_int_milliseconds
 
 logger = logging.getLogger("hishel.storages")
 
-__all__ = ("FileStorage", "RedisStorage", "SQLiteStorage", "InMemoryStorage")
+__all__ = ("FileStorage", "RedisStorage", "SQLiteStorage", "InMemoryStorage", "S3Storage")
 
 StoredResponse: TypeAlias = tp.Tuple[Response, Request, Metadata]
 
@@ -61,6 +67,9 @@ class FileStorage(BaseStorage):
     :type base_path: tp.Optional[Path], optional
     :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
     :type ttl: tp.Optional[tp.Union[int, float]], optional
+    :param check_ttl_every: How often in seconds to check staleness of **all** cache files.
+        Makes sense only with set `ttl`, defaults to 60
+    :type check_ttl_every: tp.Union[int, float]
     """
 
     def __init__(
@@ -68,6 +77,7 @@ class FileStorage(BaseStorage):
         serializer: tp.Optional[BaseSerializer] = None,
         base_path: tp.Optional[Path] = None,
         ttl: tp.Optional[tp.Union[int, float]] = None,
+        check_ttl_every: tp.Union[int, float] = 60,
     ) -> None:
         super().__init__(serializer, ttl)
 
@@ -78,6 +88,8 @@ class FileStorage(BaseStorage):
 
         self._file_manager = FileManager(is_binary=self._serializer.is_binary)
         self._lock = Lock()
+        self._check_ttl_every = check_ttl_every
+        self._last_cleaned = time.monotonic()
 
     def store(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
         """
@@ -99,7 +111,7 @@ class FileStorage(BaseStorage):
                 str(response_path),
                 self._serializer.dumps(response=response, request=request, metadata=metadata),
             )
-        self._remove_expired_caches()
+        self._remove_expired_caches(response_path)
 
     def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
         """
@@ -113,7 +125,7 @@ class FileStorage(BaseStorage):
 
         response_path = self._base_path / key
 
-        self._remove_expired_caches()
+        self._remove_expired_caches(response_path)
         with self._lock:
             if response_path.exists():
                 return self._serializer.loads(self._file_manager.read_from(str(response_path)))
@@ -122,10 +134,18 @@ class FileStorage(BaseStorage):
     def close(self) -> None:  # pragma: no cover
         return
 
-    def _remove_expired_caches(self) -> None:
+    def _remove_expired_caches(self, response_path: Path) -> None:
         if self._ttl is None:
             return
 
+        if time.monotonic() - self._last_cleaned < self._check_ttl_every:
+            if response_path.is_file():
+                age = time.time() - response_path.stat().st_mtime
+                if age > self._ttl:
+                    response_path.unlink()
+            return
+
+        self._last_cleaned = time.monotonic()
         with self._lock:
             for file in self._base_path.iterdir():
                 if file.is_file():
@@ -402,3 +422,89 @@ class InMemoryStorage(BaseStorage):
 
             for key in keys_to_remove:
                 self._cache.remove_key(key)
+
+
+class S3Storage(BaseStorage):  # pragma: no cover
+    """
+    AWS S3 storage.
+
+    :param bucket_name: The name of the bucket to store the responses in
+    :type bucket_name: str
+    :param serializer: Serializer capable of serializing and de-serializing http responses, defaults to None
+    :type serializer: tp.Optional[BaseSerializer], optional
+    :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
+    :type ttl: tp.Optional[tp.Union[int, float]], optional
+    :param client: A client for S3, defaults to None
+    :type client: tp.Optional[tp.Any], optional
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        serializer: tp.Optional[BaseSerializer] = None,
+        ttl: tp.Optional[tp.Union[int, float]] = None,
+        client: tp.Optional[tp.Any] = None,
+    ) -> None:
+        super().__init__(serializer, ttl)
+
+        if boto3 is None:  # pragma: no cover
+            raise RuntimeError(
+                (
+                    f"The `{type(self).__name__}` was used, but the required packages were not found. "
+                    "Check that you have `Hishel` installed with the `s3` extension as shown.\n"
+                    "```pip install hishel[s3]```"
+                )
+            )
+
+        self._bucket_name = bucket_name
+        client = client or boto3.client("s3")
+        self._s3_manager = S3Manager(client=client, bucket_name=bucket_name, is_binary=self._serializer.is_binary)
+        self._lock = Lock()
+
+    def store(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
+        """
+        Stores the response in the cache.
+
+        :param key: Hashed value of concatenated HTTP method and URI
+        :type key: str
+        :param response: An HTTP response
+        :type response: httpcore.Response
+        :param request: An HTTP request
+        :type request: httpcore.Request
+        :param metadata: Additioal information about the stored response
+        :type metadata: Metadata`
+        """
+
+        with self._lock:
+            serialized = self._serializer.dumps(response=response, request=request, metadata=metadata)
+            self._s3_manager.write_to(path=key, data=serialized)
+
+        self._remove_expired_caches()
+
+    def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
+        """
+        Retreives the response from the cache using his key.
+
+        :param key: Hashed value of concatenated HTTP method and URI
+        :type key: str
+        :return: An HTTP response and its HTTP request.
+        :rtype: tp.Optional[StoredResponse]
+        """
+
+        self._remove_expired_caches()
+        with self._lock:
+            try:
+                return self._serializer.loads(self._s3_manager.read_from(path=key))
+            except Exception:
+                return None
+
+    def close(self) -> None:  # pragma: no cover
+        return
+
+    def _remove_expired_caches(self) -> None:
+        if self._ttl is None:
+            return
+
+        with self._lock:
+            converted_ttl = float_seconds_to_int_milliseconds(self._ttl)
+            self._s3_manager.remove_expired(ttl=converted_ttl)

{hishel-0.0.21 → hishel-0.0.22}/pyproject.toml

@@ -29,7 +29,8 @@ classifiers = [
     "Topic :: Internet :: WWW/HTTP",
 ]
 dependencies = [
-    "httpx>=0.22.0"
+    "httpx>=0.22.0",
+    "typing_extensions>=4.8.0"
 ]
 
 [project.optional-dependencies]
@@ -46,6 +47,10 @@ sqlite = [
     "anysqlite>=0.0.5"
 ]
 
+s3 = [
+    "boto3>=1.15.0,<=1.15.3"
+]
+
 [project.urls]
 Homepage = "https://hishel.com"
 Source = "https://github.com/karpetrosyan/hishel"
@@ -88,7 +93,8 @@ filterwarnings = []
 [tool.coverage.run]
 omit = [
     "venv/*", 
-    "hishel/_sync/*"
+    "hishel/_sync/*",
+    "hishel/_s3.py"
 ]
 include = ["hishel/*", "tests/*"]