hishel 0.1.4__py3-none-any.whl → 1.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. hishel/__init__.py +55 -53
  2. hishel/{beta/_async_cache.py → _async_cache.py} +3 -3
  3. hishel/{beta → _core}/__init__.py +6 -6
  4. hishel/{beta/_core → _core}/_async/_storages/_sqlite.py +3 -3
  5. hishel/{beta/_core → _core}/_base/_storages/_base.py +13 -1
  6. hishel/{beta/_core → _core}/_base/_storages/_packing.py +5 -5
  7. hishel/_core/_headers.py +636 -0
  8. hishel/{beta/_core → _core}/_spec.py +89 -2
  9. hishel/{beta/_core → _core}/_sync/_storages/_sqlite.py +3 -3
  10. hishel/{beta/_core → _core}/models.py +1 -1
  11. hishel/{beta/_sync_cache.py → _sync_cache.py} +3 -3
  12. hishel/{beta/httpx.py → httpx.py} +18 -7
  13. hishel/{beta/requests.py → requests.py} +15 -10
  14. hishel-1.0.0.dev0.dist-info/METADATA +321 -0
  15. hishel-1.0.0.dev0.dist-info/RECORD +19 -0
  16. hishel/_async/__init__.py +0 -5
  17. hishel/_async/_client.py +0 -30
  18. hishel/_async/_mock.py +0 -43
  19. hishel/_async/_pool.py +0 -201
  20. hishel/_async/_storages.py +0 -768
  21. hishel/_async/_transports.py +0 -282
  22. hishel/_controller.py +0 -581
  23. hishel/_exceptions.py +0 -10
  24. hishel/_files.py +0 -54
  25. hishel/_headers.py +0 -215
  26. hishel/_lfu_cache.py +0 -71
  27. hishel/_lmdb_types_.pyi +0 -53
  28. hishel/_s3.py +0 -122
  29. hishel/_serializers.py +0 -329
  30. hishel/_sync/__init__.py +0 -5
  31. hishel/_sync/_client.py +0 -30
  32. hishel/_sync/_mock.py +0 -43
  33. hishel/_sync/_pool.py +0 -201
  34. hishel/_sync/_storages.py +0 -768
  35. hishel/_sync/_transports.py +0 -282
  36. hishel/_synchronization.py +0 -37
  37. hishel/beta/_core/__init__.py +0 -0
  38. hishel/beta/_core/_headers.py +0 -301
  39. hishel-0.1.4.dist-info/METADATA +0 -404
  40. hishel-0.1.4.dist-info/RECORD +0 -41
  41. {hishel-0.1.4.dist-info → hishel-1.0.0.dev0.dist-info}/WHEEL +0 -0
  42. {hishel-0.1.4.dist-info → hishel-1.0.0.dev0.dist-info}/licenses/LICENSE +0 -0
hishel/_sync/_storages.py DELETED
@@ -1,768 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import datetime
4
- import logging
5
- import os
6
- import time
7
- import typing as tp
8
- import warnings
9
- from copy import deepcopy
10
- from pathlib import Path
11
-
12
- try:
13
- import boto3
14
-
15
- from .._s3 import S3Manager
16
- except ImportError: # pragma: no cover
17
- boto3 = None # type: ignore
18
-
19
- try:
20
- import sqlite3
21
- except ImportError: # pragma: no cover
22
- sqlite3 = None # type: ignore
23
-
24
- from httpcore import Request, Response
25
-
26
- if tp.TYPE_CHECKING: # pragma: no cover
27
- from typing_extensions import TypeAlias
28
-
29
- from .._files import FileManager
30
- from .._serializers import BaseSerializer, JSONSerializer, Metadata, clone_model
31
- from .._synchronization import Lock
32
- from .._utils import float_seconds_to_int_milliseconds
33
-
34
- logger = logging.getLogger("hishel.storages")
35
-
36
- __all__ = (
37
- "BaseStorage",
38
- "FileStorage",
39
- "InMemoryStorage",
40
- "RedisStorage",
41
- "S3Storage",
42
- "SQLiteStorage",
43
- )
44
-
45
- StoredResponse: TypeAlias = tp.Tuple[Response, Request, Metadata]
46
- RemoveTypes = tp.Union[str, Response]
47
-
48
- try:
49
- import redis
50
- except ImportError: # pragma: no cover
51
- redis = None # type: ignore
52
-
53
-
54
- class BaseStorage:
55
- def __init__(
56
- self,
57
- serializer: tp.Optional[BaseSerializer] = None,
58
- ttl: tp.Optional[tp.Union[int, float]] = None,
59
- ) -> None:
60
- self._serializer = serializer or JSONSerializer()
61
- self._ttl = ttl
62
-
63
- def store(self, key: str, response: Response, request: Request, metadata: Metadata | None = None) -> None:
64
- raise NotImplementedError()
65
-
66
- def remove(self, key: RemoveTypes) -> None:
67
- raise NotImplementedError()
68
-
69
- def update_metadata(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
70
- raise NotImplementedError()
71
-
72
- def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
73
- raise NotImplementedError()
74
-
75
- def close(self) -> None:
76
- raise NotImplementedError()
77
-
78
-
79
- class FileStorage(BaseStorage):
80
- """
81
- A simple file storage.
82
-
83
- :param serializer: Serializer capable of serializing and de-serializing http responses, defaults to None
84
- :type serializer: tp.Optional[BaseSerializer], optional
85
- :param base_path: A storage base path where the responses should be saved, defaults to None
86
- :type base_path: tp.Optional[Path], optional
87
- :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
88
- :type ttl: tp.Optional[tp.Union[int, float]], optional
89
- :param check_ttl_every: How often in seconds to check staleness of **all** cache files.
90
- Makes sense only with set `ttl`, defaults to 60
91
- :type check_ttl_every: tp.Union[int, float]
92
- """
93
-
94
- def __init__(
95
- self,
96
- serializer: tp.Optional[BaseSerializer] = None,
97
- base_path: tp.Optional[Path] = None,
98
- ttl: tp.Optional[tp.Union[int, float]] = None,
99
- check_ttl_every: tp.Union[int, float] = 60,
100
- ) -> None:
101
- super().__init__(serializer, ttl)
102
-
103
- self._base_path = Path(base_path) if base_path is not None else Path(".cache/hishel")
104
- self._gitignore_file = self._base_path / ".gitignore"
105
-
106
- self._base_path.mkdir(parents=True, exist_ok=True)
107
-
108
- if not self._gitignore_file.is_file():
109
- with open(self._gitignore_file, "w", encoding="utf-8") as f:
110
- f.write("# Automatically created by Hishel\n*")
111
-
112
- self._file_manager = FileManager(is_binary=self._serializer.is_binary)
113
- self._lock = Lock()
114
- self._check_ttl_every = check_ttl_every
115
- self._last_cleaned = time.monotonic()
116
-
117
- def store(self, key: str, response: Response, request: Request, metadata: Metadata | None = None) -> None:
118
- """
119
- Stores the response in the cache.
120
-
121
- :param key: Hashed value of concatenated HTTP method and URI
122
- :type key: str
123
- :param response: An HTTP response
124
- :type response: httpcore.Response
125
- :param request: An HTTP request
126
- :type request: httpcore.Request
127
- :param metadata: Additional information about the stored response
128
- :type metadata: Optional[Metadata]
129
- """
130
-
131
- metadata = metadata or Metadata(
132
- cache_key=key, created_at=datetime.datetime.now(datetime.timezone.utc), number_of_uses=0
133
- )
134
- response_path = self._base_path / key
135
-
136
- with self._lock:
137
- self._file_manager.write_to(
138
- str(response_path),
139
- self._serializer.dumps(response=response, request=request, metadata=metadata),
140
- )
141
- self._remove_expired_caches(response_path)
142
-
143
- def remove(self, key: RemoveTypes) -> None:
144
- """
145
- Removes the response from the cache.
146
-
147
- :param key: Hashed value of concatenated HTTP method and URI or an HTTP response
148
- :type key: Union[str, Response]
149
- """
150
-
151
- if isinstance(key, Response): # pragma: no cover
152
- key = tp.cast(str, key.extensions["cache_metadata"]["cache_key"])
153
-
154
- response_path = self._base_path / key
155
-
156
- with self._lock:
157
- if response_path.exists():
158
- response_path.unlink(missing_ok=True)
159
-
160
- def update_metadata(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
161
- """
162
- Updates the metadata of the stored response.
163
-
164
- :param key: Hashed value of concatenated HTTP method and URI
165
- :type key: str
166
- :param response: An HTTP response
167
- :type response: httpcore.Response
168
- :param request: An HTTP request
169
- :type request: httpcore.Request
170
- :param metadata: Additional information about the stored response
171
- :type metadata: Metadata
172
- """
173
- response_path = self._base_path / key
174
-
175
- with self._lock:
176
- if response_path.exists():
177
- atime = response_path.stat().st_atime
178
- old_mtime = response_path.stat().st_mtime
179
- self._file_manager.write_to(
180
- str(response_path),
181
- self._serializer.dumps(response=response, request=request, metadata=metadata),
182
- )
183
-
184
- # Restore the old atime and mtime (we use mtime to check the cache expiration time)
185
- os.utime(response_path, (atime, old_mtime))
186
- return
187
-
188
- return self.store(key, response, request, metadata) # pragma: no cover
189
-
190
- def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
191
- """
192
- Retreives the response from the cache using his key.
193
-
194
- :param key: Hashed value of concatenated HTTP method and URI
195
- :type key: str
196
- :return: An HTTP response and his HTTP request.
197
- :rtype: tp.Optional[StoredResponse]
198
- """
199
-
200
- response_path = self._base_path / key
201
-
202
- self._remove_expired_caches(response_path)
203
- with self._lock:
204
- if response_path.exists():
205
- read_data = self._file_manager.read_from(str(response_path))
206
- if len(read_data) != 0:
207
- return self._serializer.loads(read_data)
208
- return None
209
-
210
- def close(self) -> None: # pragma: no cover
211
- return
212
-
213
- def _remove_expired_caches(self, response_path: Path) -> None:
214
- if self._ttl is None:
215
- return
216
-
217
- if time.monotonic() - self._last_cleaned < self._check_ttl_every:
218
- if response_path.is_file():
219
- age = time.time() - response_path.stat().st_mtime
220
- if age > self._ttl:
221
- response_path.unlink(missing_ok=True)
222
- return
223
-
224
- self._last_cleaned = time.monotonic()
225
- with self._lock:
226
- with os.scandir(self._base_path) as entries:
227
- for entry in entries:
228
- try:
229
- if entry.is_file():
230
- age = time.time() - entry.stat().st_mtime
231
- if age > self._ttl:
232
- os.unlink(entry.path)
233
- except FileNotFoundError: # pragma: no cover
234
- pass
235
-
236
-
237
- class SQLiteStorage(BaseStorage):
238
- """
239
- A simple sqlite3 storage.
240
-
241
- :param serializer: Serializer capable of serializing and de-serializing http responses, defaults to None
242
- :type serializer: tp.Optional[BaseSerializer], optional
243
- :param connection: A connection for sqlite, defaults to None
244
- :type connection: tp.Optional[sqlite3.Connection], optional
245
- :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
246
- :type ttl: tp.Optional[tp.Union[int, float]], optional
247
- """
248
-
249
- def __init__(
250
- self,
251
- serializer: tp.Optional[BaseSerializer] = None,
252
- connection: tp.Optional[sqlite3.Connection] = None,
253
- ttl: tp.Optional[tp.Union[int, float]] = None,
254
- ) -> None:
255
- if sqlite3 is None: # pragma: no cover
256
- raise RuntimeError(
257
- f"The `{type(self).__name__}` was used, but the required packages were not found. "
258
- "Check that you have `Hishel` installed with the `sqlite` extension as shown.\n"
259
- "```pip install hishel[sqlite]```"
260
- )
261
- super().__init__(serializer, ttl)
262
-
263
- self._connection: tp.Optional[sqlite3.Connection] = connection or None
264
- self._setup_lock = Lock()
265
- self._setup_completed: bool = False
266
- self._lock = Lock()
267
-
268
- def _setup(self) -> None:
269
- with self._setup_lock:
270
- if not self._setup_completed:
271
- if not self._connection: # pragma: no cover
272
- self._connection = sqlite3.connect(".hishel.sqlite", check_same_thread=False)
273
- self._connection.execute(
274
- "CREATE TABLE IF NOT EXISTS cache(key TEXT, data BLOB, date_created REAL)"
275
- )
276
- self._connection.commit()
277
- self._setup_completed = True
278
-
279
- def store(self, key: str, response: Response, request: Request, metadata: Metadata | None = None) -> None:
280
- """
281
- Stores the response in the cache.
282
-
283
- :param key: Hashed value of concatenated HTTP method and URI
284
- :type key: str
285
- :param response: An HTTP response
286
- :type response: httpcore.Response
287
- :param request: An HTTP request
288
- :type request: httpcore.Request
289
- :param metadata: Additioal information about the stored response
290
- :type metadata: Optional[Metadata]
291
- """
292
-
293
- self._setup()
294
- assert self._connection
295
-
296
- metadata = metadata or Metadata(
297
- cache_key=key, created_at=datetime.datetime.now(datetime.timezone.utc), number_of_uses=0
298
- )
299
-
300
- with self._lock:
301
- self._connection.execute("DELETE FROM cache WHERE key = ?", [key])
302
- serialized_response = self._serializer.dumps(response=response, request=request, metadata=metadata)
303
- self._connection.execute(
304
- "INSERT INTO cache(key, data, date_created) VALUES(?, ?, ?)", [key, serialized_response, time.time()]
305
- )
306
- self._connection.commit()
307
- self._remove_expired_caches()
308
-
309
- def remove(self, key: RemoveTypes) -> None:
310
- """
311
- Removes the response from the cache.
312
-
313
- :param key: Hashed value of concatenated HTTP method and URI or an HTTP response
314
- :type key: Union[str, Response]
315
- """
316
-
317
- self._setup()
318
- assert self._connection
319
-
320
- if isinstance(key, Response): # pragma: no cover
321
- key = tp.cast(str, key.extensions["cache_metadata"]["cache_key"])
322
-
323
- with self._lock:
324
- self._connection.execute("DELETE FROM cache WHERE key = ?", [key])
325
- self._connection.commit()
326
-
327
- def update_metadata(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
328
- """
329
- Updates the metadata of the stored response.
330
-
331
- :param key: Hashed value of concatenated HTTP method and URI
332
- :type key: str
333
- :param response: An HTTP response
334
- :type response: httpcore.Response
335
- :param request: An HTTP request
336
- :type request: httpcore.Request
337
- :param metadata: Additional information about the stored response
338
- :type metadata: Metadata
339
- """
340
-
341
- self._setup()
342
- assert self._connection
343
-
344
- with self._lock:
345
- cursor = self._connection.execute("SELECT data FROM cache WHERE key = ?", [key])
346
- row = cursor.fetchone()
347
- if row is not None:
348
- serialized_response = self._serializer.dumps(response=response, request=request, metadata=metadata)
349
- self._connection.execute("UPDATE cache SET data = ? WHERE key = ?", [serialized_response, key])
350
- self._connection.commit()
351
- return
352
- return self.store(key, response, request, metadata) # pragma: no cover
353
-
354
- def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
355
- """
356
- Retreives the response from the cache using his key.
357
-
358
- :param key: Hashed value of concatenated HTTP method and URI
359
- :type key: str
360
- :return: An HTTP response and its HTTP request.
361
- :rtype: tp.Optional[StoredResponse]
362
- """
363
-
364
- self._setup()
365
- assert self._connection
366
-
367
- self._remove_expired_caches()
368
- with self._lock:
369
- cursor = self._connection.execute("SELECT data FROM cache WHERE key = ?", [key])
370
- row = cursor.fetchone()
371
- if row is None:
372
- return None
373
-
374
- cached_response = row[0]
375
- return self._serializer.loads(cached_response)
376
-
377
- def close(self) -> None: # pragma: no cover
378
- if self._connection is not None:
379
- self._connection.close()
380
-
381
- def _remove_expired_caches(self) -> None:
382
- assert self._connection
383
- if self._ttl is None:
384
- return
385
-
386
- with self._lock:
387
- self._connection.execute("DELETE FROM cache WHERE date_created + ? < ?", [self._ttl, time.time()])
388
- self._connection.commit()
389
-
390
-
391
- class RedisStorage(BaseStorage):
392
- """
393
- A simple redis storage.
394
-
395
- :param serializer: Serializer capable of serializing and de-serializing http responses, defaults to None
396
- :type serializer: tp.Optional[BaseSerializer], optional
397
- :param client: A client for redis, defaults to None
398
- :type client: tp.Optional["redis.Redis"], optional
399
- :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
400
- :type ttl: tp.Optional[tp.Union[int, float]], optional
401
- """
402
-
403
- def __init__(
404
- self,
405
- serializer: tp.Optional[BaseSerializer] = None,
406
- client: tp.Optional[redis.Redis] = None, # type: ignore
407
- ttl: tp.Optional[tp.Union[int, float]] = None,
408
- ) -> None:
409
- if redis is None: # pragma: no cover
410
- raise RuntimeError(
411
- f"The `{type(self).__name__}` was used, but the required packages were not found. "
412
- "Check that you have `Hishel` installed with the `redis` extension as shown.\n"
413
- "```pip install hishel[redis]```"
414
- )
415
- super().__init__(serializer, ttl)
416
-
417
- if client is None:
418
- self._client = redis.Redis() # type: ignore
419
- else: # pragma: no cover
420
- self._client = client
421
-
422
- def store(self, key: str, response: Response, request: Request, metadata: Metadata | None = None) -> None:
423
- """
424
- Stores the response in the cache.
425
-
426
- :param key: Hashed value of concatenated HTTP method and URI
427
- :type key: str
428
- :param response: An HTTP response
429
- :type response: httpcore.Response
430
- :param request: An HTTP request
431
- :type request: httpcore.Request
432
- :param metadata: Additioal information about the stored response
433
- :type metadata: Optional[Metadata]
434
- """
435
-
436
- metadata = metadata or Metadata(
437
- cache_key=key, created_at=datetime.datetime.now(datetime.timezone.utc), number_of_uses=0
438
- )
439
-
440
- if self._ttl is not None:
441
- px = float_seconds_to_int_milliseconds(self._ttl)
442
- else:
443
- px = None
444
-
445
- self._client.set(
446
- key, self._serializer.dumps(response=response, request=request, metadata=metadata), px=px
447
- )
448
-
449
- def remove(self, key: RemoveTypes) -> None:
450
- """
451
- Removes the response from the cache.
452
-
453
- :param key: Hashed value of concatenated HTTP method and URI or an HTTP response
454
- :type key: Union[str, Response]
455
- """
456
-
457
- if isinstance(key, Response): # pragma: no cover
458
- key = tp.cast(str, key.extensions["cache_metadata"]["cache_key"])
459
-
460
- self._client.delete(key)
461
-
462
- def update_metadata(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
463
- """
464
- Updates the metadata of the stored response.
465
-
466
- :param key: Hashed value of concatenated HTTP method and URI
467
- :type key: str
468
- :param response: An HTTP response
469
- :type response: httpcore.Response
470
- :param request: An HTTP request
471
- :type request: httpcore.Request
472
- :param metadata: Additional information about the stored response
473
- :type metadata: Metadata
474
- """
475
-
476
- ttl_in_milliseconds = self._client.pttl(key)
477
-
478
- # -2: if the key does not exist in Redis
479
- # -1: if the key exists in Redis but has no expiration
480
- if ttl_in_milliseconds == -2 or ttl_in_milliseconds == -1: # pragma: no cover
481
- self.store(key, response, request, metadata)
482
- else:
483
- self._client.set(
484
- key,
485
- self._serializer.dumps(response=response, request=request, metadata=metadata),
486
- px=ttl_in_milliseconds,
487
- )
488
-
489
- def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
490
- """
491
- Retreives the response from the cache using his key.
492
-
493
- :param key: Hashed value of concatenated HTTP method and URI
494
- :type key: str
495
- :return: An HTTP response and its HTTP request.
496
- :rtype: tp.Optional[StoredResponse]
497
- """
498
-
499
- cached_response = self._client.get(key)
500
- if cached_response is None:
501
- return None
502
-
503
- return self._serializer.loads(cached_response)
504
-
505
- def close(self) -> None: # pragma: no cover
506
- self._client.close()
507
-
508
-
509
- class InMemoryStorage(BaseStorage):
510
- """
511
- A simple in-memory storage.
512
-
513
- :param serializer: Serializer capable of serializing and de-serializing http responses, defaults to None
514
- :type serializer: tp.Optional[BaseSerializer], optional
515
- :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
516
- :type ttl: tp.Optional[tp.Union[int, float]], optional
517
- :param capacity: The maximum number of responses that can be cached, defaults to 128
518
- :type capacity: int, optional
519
- """
520
-
521
- def __init__(
522
- self,
523
- serializer: tp.Optional[BaseSerializer] = None,
524
- ttl: tp.Optional[tp.Union[int, float]] = None,
525
- capacity: int = 128,
526
- ) -> None:
527
- super().__init__(serializer, ttl)
528
-
529
- if serializer is not None: # pragma: no cover
530
- warnings.warn("The serializer is not used in the in-memory storage.", RuntimeWarning)
531
-
532
- from hishel import LFUCache
533
-
534
- self._cache: LFUCache[str, tp.Tuple[StoredResponse, float]] = LFUCache(capacity=capacity)
535
- self._lock = Lock()
536
-
537
- def store(self, key: str, response: Response, request: Request, metadata: Metadata | None = None) -> None:
538
- """
539
- Stores the response in the cache.
540
-
541
- :param key: Hashed value of concatenated HTTP method and URI
542
- :type key: str
543
- :param response: An HTTP response
544
- :type response: httpcore.Response
545
- :param request: An HTTP request
546
- :type request: httpcore.Request
547
- :param metadata: Additioal information about the stored response
548
- :type metadata: Optional[Metadata]
549
- """
550
-
551
- metadata = metadata or Metadata(
552
- cache_key=key, created_at=datetime.datetime.now(datetime.timezone.utc), number_of_uses=0
553
- )
554
-
555
- with self._lock:
556
- response_clone = clone_model(response)
557
- request_clone = clone_model(request)
558
- stored_response: StoredResponse = (deepcopy(response_clone), deepcopy(request_clone), metadata)
559
- self._cache.put(key, (stored_response, time.monotonic()))
560
- self._remove_expired_caches()
561
-
562
- def remove(self, key: RemoveTypes) -> None:
563
- """
564
- Removes the response from the cache.
565
-
566
- :param key: Hashed value of concatenated HTTP method and URI or an HTTP response
567
- :type key: Union[str, Response]
568
- """
569
-
570
- if isinstance(key, Response): # pragma: no cover
571
- key = tp.cast(str, key.extensions["cache_metadata"]["cache_key"])
572
-
573
- with self._lock:
574
- self._cache.remove_key(key)
575
-
576
- def update_metadata(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
577
- """
578
- Updates the metadata of the stored response.
579
-
580
- :param key: Hashed value of concatenated HTTP method and URI
581
- :type key: str
582
- :param response: An HTTP response
583
- :type response: httpcore.Response
584
- :param request: An HTTP request
585
- :type request: httpcore.Request
586
- :param metadata: Additional information about the stored response
587
- :type metadata: Metadata
588
- """
589
-
590
- with self._lock:
591
- try:
592
- stored_response, created_at = self._cache.get(key)
593
- stored_response = (stored_response[0], stored_response[1], metadata)
594
- self._cache.put(key, (stored_response, created_at))
595
- return
596
- except KeyError: # pragma: no cover
597
- pass
598
- self.store(key, response, request, metadata) # pragma: no cover
599
-
600
- def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
601
- """
602
- Retreives the response from the cache using his key.
603
-
604
- :param key: Hashed value of concatenated HTTP method and URI
605
- :type key: str
606
- :return: An HTTP response and its HTTP request.
607
- :rtype: tp.Optional[StoredResponse]
608
- """
609
-
610
- self._remove_expired_caches()
611
- with self._lock:
612
- try:
613
- stored_response, _ = self._cache.get(key)
614
- except KeyError:
615
- return None
616
- return stored_response
617
-
618
- def close(self) -> None: # pragma: no cover
619
- return
620
-
621
- def _remove_expired_caches(self) -> None:
622
- if self._ttl is None:
623
- return
624
-
625
- with self._lock:
626
- keys_to_remove = set()
627
-
628
- for key in self._cache:
629
- created_at = self._cache.get(key)[1]
630
-
631
- if time.monotonic() - created_at > self._ttl:
632
- keys_to_remove.add(key)
633
-
634
- for key in keys_to_remove:
635
- self._cache.remove_key(key)
636
-
637
-
638
- class S3Storage(BaseStorage): # pragma: no cover
639
- """
640
- AWS S3 storage.
641
-
642
- :param bucket_name: The name of the bucket to store the responses in
643
- :type bucket_name: str
644
- :param serializer: Serializer capable of serializing and de-serializing http responses, defaults to None
645
- :type serializer: tp.Optional[BaseSerializer], optional
646
- :param ttl: Specifies the maximum number of seconds that the response can be cached, defaults to None
647
- :type ttl: tp.Optional[tp.Union[int, float]], optional
648
- :param check_ttl_every: How often in seconds to check staleness of **all** cache files.
649
- Makes sense only with set `ttl`, defaults to 60
650
- :type check_ttl_every: tp.Union[int, float]
651
- :param client: A client for S3, defaults to None
652
- :type client: tp.Optional[tp.Any], optional
653
- :param path_prefix: A path prefix to use for S3 object keys, defaults to "hishel-"
654
- :type path_prefix: str, optional
655
- """
656
-
657
- def __init__(
658
- self,
659
- bucket_name: str,
660
- serializer: tp.Optional[BaseSerializer] = None,
661
- ttl: tp.Optional[tp.Union[int, float]] = None,
662
- check_ttl_every: tp.Union[int, float] = 60,
663
- client: tp.Optional[tp.Any] = None,
664
- path_prefix: str = "hishel-",
665
- ) -> None:
666
- super().__init__(serializer, ttl)
667
-
668
- if boto3 is None: # pragma: no cover
669
- raise RuntimeError(
670
- f"The `{type(self).__name__}` was used, but the required packages were not found. "
671
- "Check that you have `Hishel` installed with the `s3` extension as shown.\n"
672
- "```pip install hishel[s3]```"
673
- )
674
-
675
- self._bucket_name = bucket_name
676
- client = client or boto3.client("s3")
677
- self._s3_manager = S3Manager(
678
- client=client,
679
- bucket_name=bucket_name,
680
- is_binary=self._serializer.is_binary,
681
- check_ttl_every=check_ttl_every,
682
- path_prefix=path_prefix,
683
- )
684
- self._lock = Lock()
685
-
686
- def store(self, key: str, response: Response, request: Request, metadata: Metadata | None = None) -> None:
687
- """
688
- Stores the response in the cache.
689
-
690
- :param key: Hashed value of concatenated HTTP method and URI
691
- :type key: str
692
- :param response: An HTTP response
693
- :type response: httpcore.Response
694
- :param request: An HTTP request
695
- :type request: httpcore.Request
696
- :param metadata: Additioal information about the stored response
697
- :type metadata: Optional[Metadata]`
698
- """
699
-
700
- metadata = metadata or Metadata(
701
- cache_key=key, created_at=datetime.datetime.now(datetime.timezone.utc), number_of_uses=0
702
- )
703
-
704
- with self._lock:
705
- serialized = self._serializer.dumps(response=response, request=request, metadata=metadata)
706
- self._s3_manager.write_to(path=key, data=serialized)
707
-
708
- self._remove_expired_caches(key)
709
-
710
- def remove(self, key: RemoveTypes) -> None:
711
- """
712
- Removes the response from the cache.
713
-
714
- :param key: Hashed value of concatenated HTTP method and URI or an HTTP response
715
- :type key: Union[str, Response]
716
- """
717
-
718
- if isinstance(key, Response): # pragma: no cover
719
- key = tp.cast(str, key.extensions["cache_metadata"]["cache_key"])
720
-
721
- with self._lock:
722
- self._s3_manager.remove_entry(key)
723
-
724
- def update_metadata(self, key: str, response: Response, request: Request, metadata: Metadata) -> None:
725
- """
726
- Updates the metadata of the stored response.
727
-
728
- :param key: Hashed value of concatenated HTTP method and URI
729
- :type key: str
730
- :param response: An HTTP response
731
- :type response: httpcore.Response
732
- :param request: An HTTP request
733
- :type request: httpcore.Request
734
- :param metadata: Additional information about the stored response
735
- :type metadata: Metadata
736
- """
737
-
738
- with self._lock:
739
- serialized = self._serializer.dumps(response=response, request=request, metadata=metadata)
740
- self._s3_manager.write_to(path=key, data=serialized, only_metadata=True)
741
-
742
- def retrieve(self, key: str) -> tp.Optional[StoredResponse]:
743
- """
744
- Retreives the response from the cache using his key.
745
-
746
- :param key: Hashed value of concatenated HTTP method and URI
747
- :type key: str
748
- :return: An HTTP response and its HTTP request.
749
- :rtype: tp.Optional[StoredResponse]
750
- """
751
-
752
- self._remove_expired_caches(key)
753
- with self._lock:
754
- try:
755
- return self._serializer.loads(self._s3_manager.read_from(path=key))
756
- except Exception:
757
- return None
758
-
759
- def close(self) -> None: # pragma: no cover
760
- return
761
-
762
- def _remove_expired_caches(self, key: str) -> None:
763
- if self._ttl is None:
764
- return
765
-
766
- with self._lock:
767
- converted_ttl = float_seconds_to_int_milliseconds(self._ttl)
768
- self._s3_manager.remove_expired(ttl=converted_ttl, key=key)