xtr-cache 1.2.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/PKG-INFO +32 -3
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/README.md +29 -2
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/pyproject.toml +6 -1
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/pyproject.toml.orig +11 -1
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/chain_adapter.py +10 -2
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/deferred_items_mixin.py +7 -2
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/filesystem_adapter.py +96 -7
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/tag_aware_adapter.py +6 -3
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/bundle/cache_config.py +1 -2
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/lock_registry.py +18 -12
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/LICENSE +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/__init__.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/__init__.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/abstract_adapter.py +1 -1
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/adapter_factory.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/adapter_interface.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/array_adapter.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/contracts_mixin.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/null_adapter.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/redis_adapter.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/tag_aware_adapter_interface.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/adapter/tagged_value.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/bundle/__init__.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/bundle/cache_bundle.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/bundle/pool_config.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/cache_item.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/cache_pool_clearer.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/__init__.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/cache_pool_clear_command.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/cache_pool_delete_command.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/cache_pool_invalidate_tags_command.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/cache_pool_list_command.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/cache_pool_prune_command.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/pool_command.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/pools.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/exception/__init__.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/exception/logic_error.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/exception/marshalling_error.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/marshaller/__init__.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/marshaller/default_marshaller.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/marshaller/deflate_marshaller.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/marshaller/marshaller_interface.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/marshaller/sodium_marshaller.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/pruneable_interface.py +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/py.typed +0 -0
- {xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/value_wrapper.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xtr-cache
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Cache pools in memory, in files, in Redis, chained or tag-aware, with stampede protection.
|
|
5
5
|
Keywords: cache,redis,stampede,tags,asyncio
|
|
6
6
|
Author: Razvan Ceana
|
|
@@ -23,11 +23,13 @@ Requires-Dist: xtr-logging-contracts>=1.0,<2
|
|
|
23
23
|
Requires-Dist: xtr-console>=1.0,<2 ; extra == 'console'
|
|
24
24
|
Requires-Dist: xtr-dependency-injection>=1.0,<2 ; extra == 'di'
|
|
25
25
|
Requires-Dist: xtr-service-contracts>=1.0,<2 ; extra == 'di'
|
|
26
|
+
Requires-Dist: xtr-logging>=1.0,<2 ; extra == 'logging'
|
|
26
27
|
Requires-Dist: redis>=5.0 ; extra == 'redis'
|
|
27
28
|
Requires-Dist: pynacl>=1.5 ; extra == 'sodium'
|
|
28
29
|
Requires-Python: >=3.11
|
|
29
30
|
Provides-Extra: console
|
|
30
31
|
Provides-Extra: di
|
|
32
|
+
Provides-Extra: logging
|
|
31
33
|
Provides-Extra: redis
|
|
32
34
|
Provides-Extra: sodium
|
|
33
35
|
Description-Content-Type: text/markdown
|
|
@@ -169,6 +171,11 @@ goes to a temporary file first and replaces the value's file in one step, so rea
|
|
|
169
171
|
half a value. File work runs on a worker thread, off the event loop. Nothing is created until the
|
|
170
172
|
first write. An expired file is removed when read; `await pool.prune()` removes the rest.
|
|
171
173
|
|
|
174
|
+
Given no directory, a pool uses `xtr-cache` in the system's temporary directory, which any user
|
|
175
|
+
may create first. Values are unpickled when read, so that directory is made readable by its owner
|
|
176
|
+
alone, and one belonging to another user is refused: reads miss, writes fail, and the pool logs
|
|
177
|
+
why. A directory you give is used as it is.
|
|
178
|
+
|
|
172
179
|
### `RedisAdapter`
|
|
173
180
|
|
|
174
181
|
```python
|
|
@@ -183,7 +190,9 @@ await pool.aclose() # ... which it closes
|
|
|
183
190
|
Each value is a string key, `namespace:key`, expired by the server itself. Reads fetch many keys
|
|
184
191
|
in one round trip, writes are pipelined, and clearing scans the namespace and unlinks its keys in
|
|
185
192
|
batches — so set a namespace when the database holds anything else: without one, clearing empties
|
|
186
|
-
the database.
|
|
193
|
+
the database. A client built from a DSN gives up on a server silent for 5 seconds — a miss, or a
|
|
194
|
+
failed write — unless `?socket_timeout=` and `?socket_connect_timeout=` in the DSN say otherwise.
|
|
195
|
+
One server; not a cluster, not Sentinel.
|
|
187
196
|
|
|
188
197
|
### `ChainAdapter`
|
|
189
198
|
|
|
@@ -278,6 +287,26 @@ pool = RedisAdapter.from_url("redis://cache", "app", marshaller=SodiumMarshaller
|
|
|
278
287
|
Keys rotate: the first encrypts, every one decrypts. Put the new key first and keep the old one
|
|
279
288
|
until what it encrypted has expired.
|
|
280
289
|
|
|
290
|
+
## Use in an application
|
|
291
|
+
|
|
292
|
+
Everything adding this package to an application on
|
|
293
|
+
[xtr-dependency-injection](../xtr-dependency-injection) takes — and, read backwards, what removing it undoes.
|
|
294
|
+
|
|
295
|
+
- **Install** — `uv add "xtr-cache[di]"`; add `redis`, `sodium`, `console` or `logging` for what
|
|
296
|
+
you use.
|
|
297
|
+
- **Activate** — `CacheBundle: {"all": True}` in `BUNDLES` in `<app>/bundles.py`, imported
|
|
298
|
+
from `xtr_cache.bundle`.
|
|
299
|
+
- **Brings along** — the logging and console bundles, when those packages are installed.
|
|
300
|
+
- **Configure** — optional: with no configuration there is one `app` pool on files. Pools,
|
|
301
|
+
adapters and the stampede lock go in `<app>/config/cache.py`, a `@configure` function
|
|
302
|
+
returning `CacheConfig` — see [Kernel / bundle](#kernel--bundle).
|
|
303
|
+
- **Environment** — nothing required. A DSN given as `env(...)` must be set when the
|
|
304
|
+
application boots: booting checks every pool.
|
|
305
|
+
- **Ignore** — `var/`: files live under `kernel.share_dir`, `var/share` in the project.
|
|
306
|
+
- **Remove** — drop the `BUNDLES` entry, delete `<app>/config/cache.py`, then
|
|
307
|
+
`uv remove xtr-cache`.
|
|
308
|
+
- **Check** — `debug:bundles` shows `cache` as `listed` and `active`.
|
|
309
|
+
|
|
281
310
|
## Kernel / bundle
|
|
282
311
|
|
|
283
312
|
With [xtr-dependency-injection](../xtr-dependency-injection), list the bundle and name the pools:
|
|
@@ -333,7 +362,7 @@ class Catalogue:
|
|
|
333
362
|
|---|---|
|
|
334
363
|
| `app` | The `app` pool's adapter, or several to chain. `"filesystem"` by default |
|
|
335
364
|
| `pools` | Every other pool: an adapter, several, or a `PoolConfig(adapter, default_lifetime, tags, namespace)`. A pool without adapter uses `app`'s. `tags=True` keeps tag versions in the pool itself; `tags="other"` keeps them in the pool named `other` |
|
|
336
|
-
| `directory` | Where `"filesystem"` keeps files: `"%kernel.share_dir%/cache"` by default,
|
|
365
|
+
| `directory` | Where `"filesystem"` keeps files: `"%kernel.share_dir%/cache"` by default, `var/share/cache` in the project |
|
|
337
366
|
| `prefix_seed` | What each pool's namespace is derived from, with its name: `"%kernel.project_dir%"` by default, so two applications on one backend never meet; give two the same seed to share values |
|
|
338
367
|
| `stampede_lock` | Where stampede locks go — any lock DSN: `"flock://%kernel.share_dir%/cache/locks"` by default, `"redis://…"` to span machines — or `None` for per-process protection only |
|
|
339
368
|
|
|
@@ -135,6 +135,11 @@ goes to a temporary file first and replaces the value's file in one step, so rea
|
|
|
135
135
|
half a value. File work runs on a worker thread, off the event loop. Nothing is created until the
|
|
136
136
|
first write. An expired file is removed when read; `await pool.prune()` removes the rest.
|
|
137
137
|
|
|
138
|
+
Given no directory, a pool uses `xtr-cache` in the system's temporary directory, which any user
|
|
139
|
+
may create first. Values are unpickled when read, so that directory is made readable by its owner
|
|
140
|
+
alone, and one belonging to another user is refused: reads miss, writes fail, and the pool logs
|
|
141
|
+
why. A directory you give is used as it is.
|
|
142
|
+
|
|
138
143
|
### `RedisAdapter`
|
|
139
144
|
|
|
140
145
|
```python
|
|
@@ -149,7 +154,9 @@ await pool.aclose() # ... which it closes
|
|
|
149
154
|
Each value is a string key, `namespace:key`, expired by the server itself. Reads fetch many keys
|
|
150
155
|
in one round trip, writes are pipelined, and clearing scans the namespace and unlinks its keys in
|
|
151
156
|
batches — so set a namespace when the database holds anything else: without one, clearing empties
|
|
152
|
-
the database.
|
|
157
|
+
the database. A client built from a DSN gives up on a server silent for 5 seconds — a miss, or a
|
|
158
|
+
failed write — unless `?socket_timeout=` and `?socket_connect_timeout=` in the DSN say otherwise.
|
|
159
|
+
One server; not a cluster, not Sentinel.
|
|
153
160
|
|
|
154
161
|
### `ChainAdapter`
|
|
155
162
|
|
|
@@ -244,6 +251,26 @@ pool = RedisAdapter.from_url("redis://cache", "app", marshaller=SodiumMarshaller
|
|
|
244
251
|
Keys rotate: the first encrypts, every one decrypts. Put the new key first and keep the old one
|
|
245
252
|
until what it encrypted has expired.
|
|
246
253
|
|
|
254
|
+
## Use in an application
|
|
255
|
+
|
|
256
|
+
Everything adding this package to an application on
|
|
257
|
+
[xtr-dependency-injection](../xtr-dependency-injection) takes — and, read backwards, what removing it undoes.
|
|
258
|
+
|
|
259
|
+
- **Install** — `uv add "xtr-cache[di]"`; add `redis`, `sodium`, `console` or `logging` for what
|
|
260
|
+
you use.
|
|
261
|
+
- **Activate** — `CacheBundle: {"all": True}` in `BUNDLES` in `<app>/bundles.py`, imported
|
|
262
|
+
from `xtr_cache.bundle`.
|
|
263
|
+
- **Brings along** — the logging and console bundles, when those packages are installed.
|
|
264
|
+
- **Configure** — optional: with no configuration there is one `app` pool on files. Pools,
|
|
265
|
+
adapters and the stampede lock go in `<app>/config/cache.py`, a `@configure` function
|
|
266
|
+
returning `CacheConfig` — see [Kernel / bundle](#kernel--bundle).
|
|
267
|
+
- **Environment** — nothing required. A DSN given as `env(...)` must be set when the
|
|
268
|
+
application boots: booting checks every pool.
|
|
269
|
+
- **Ignore** — `var/`: files live under `kernel.share_dir`, `var/share` in the project.
|
|
270
|
+
- **Remove** — drop the `BUNDLES` entry, delete `<app>/config/cache.py`, then
|
|
271
|
+
`uv remove xtr-cache`.
|
|
272
|
+
- **Check** — `debug:bundles` shows `cache` as `listed` and `active`.
|
|
273
|
+
|
|
247
274
|
## Kernel / bundle
|
|
248
275
|
|
|
249
276
|
With [xtr-dependency-injection](../xtr-dependency-injection), list the bundle and name the pools:
|
|
@@ -299,7 +326,7 @@ class Catalogue:
|
|
|
299
326
|
|---|---|
|
|
300
327
|
| `app` | The `app` pool's adapter, or several to chain. `"filesystem"` by default |
|
|
301
328
|
| `pools` | Every other pool: an adapter, several, or a `PoolConfig(adapter, default_lifetime, tags, namespace)`. A pool without adapter uses `app`'s. `tags=True` keeps tag versions in the pool itself; `tags="other"` keeps them in the pool named `other` |
|
|
302
|
-
| `directory` | Where `"filesystem"` keeps files: `"%kernel.share_dir%/cache"` by default,
|
|
329
|
+
| `directory` | Where `"filesystem"` keeps files: `"%kernel.share_dir%/cache"` by default, `var/share/cache` in the project |
|
|
303
330
|
| `prefix_seed` | What each pool's namespace is derived from, with its name: `"%kernel.project_dir%"` by default, so two applications on one backend never meet; give two the same seed to share values |
|
|
304
331
|
| `stampede_lock` | Where stampede locks go — any lock DSN: `"flock://%kernel.share_dir%/cache/locks"` by default, `"redis://…"` to span machines — or `None` for per-process protection only |
|
|
305
332
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "xtr-cache"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.4.0"
|
|
4
4
|
description = "Cache pools in memory, in files, in Redis, chained or tag-aware, with stampede protection."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -43,6 +43,10 @@ di = [
|
|
|
43
43
|
"xtr-service-contracts>=1.0,<2",
|
|
44
44
|
]
|
|
45
45
|
console = ["xtr-console>=1.0,<2"]
|
|
46
|
+
logging = ["xtr-logging>=1.0,<2"]
|
|
47
|
+
|
|
48
|
+
[project.entry-points."xtr_dependency_injection.bundles"]
|
|
49
|
+
cache = "xtr_cache.bundle:CacheBundle"
|
|
46
50
|
|
|
47
51
|
[dependency-groups]
|
|
48
52
|
dev = [
|
|
@@ -174,6 +178,7 @@ source = ["src"]
|
|
|
174
178
|
branch = true
|
|
175
179
|
|
|
176
180
|
[tool.coverage.report]
|
|
181
|
+
fail_under = 96
|
|
177
182
|
exclude_lines = [
|
|
178
183
|
"pragma: no cover",
|
|
179
184
|
"if TYPE_CHECKING:",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "xtr-cache"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.4.0"
|
|
4
4
|
description = "Cache pools in memory, in files, in Redis, chained or tag-aware, with stampede protection."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -42,6 +42,14 @@ di = [
|
|
|
42
42
|
console = [
|
|
43
43
|
"xtr-console>=1.0,<2",
|
|
44
44
|
]
|
|
45
|
+
logging = [
|
|
46
|
+
"xtr-logging>=1.0,<2",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
# Advertises the bundle so debug:bundles can name it when installed but not listed.
|
|
50
|
+
# Advertising activates nothing: an application lists the bundles it wants.
|
|
51
|
+
[project.entry-points."xtr_dependency_injection.bundles"]
|
|
52
|
+
cache = "xtr_cache.bundle:CacheBundle"
|
|
45
53
|
|
|
46
54
|
[dependency-groups]
|
|
47
55
|
dev = [
|
|
@@ -178,6 +186,8 @@ source = ["src"]
|
|
|
178
186
|
branch = true
|
|
179
187
|
|
|
180
188
|
[tool.coverage.report]
|
|
189
|
+
# CI runs the tests with --cov; a change taking coverage below this fails there.
|
|
190
|
+
fail_under = 96
|
|
181
191
|
exclude_lines = [
|
|
182
192
|
"pragma: no cover",
|
|
183
193
|
"if TYPE_CHECKING:",
|
|
@@ -167,8 +167,16 @@ class ChainAdapter(ContractsMixin, AdapterInterface, NamespacedPoolInterface, Pr
|
|
|
167
167
|
|
|
168
168
|
@override
|
|
169
169
|
def _scope(self) -> str:
|
|
170
|
-
first
|
|
171
|
-
|
|
170
|
+
"""Return the first level's scope that sets the chain apart.
|
|
171
|
+
|
|
172
|
+
A level kept in memory has no namespace, being nobody else's; the
|
|
173
|
+
levels behind it do, and name the pool as well.
|
|
174
|
+
"""
|
|
175
|
+
for adapter in self._adapters:
|
|
176
|
+
scope = adapter._scope() if isinstance(adapter, ContractsMixin) else "" # noqa: SLF001 — a pool of this library.
|
|
177
|
+
if scope:
|
|
178
|
+
return scope
|
|
179
|
+
return ""
|
|
172
180
|
|
|
173
181
|
@override
|
|
174
182
|
def __repr__(self) -> str:
|
|
@@ -23,8 +23,10 @@ class DeferredItemsMixin(CacheItemPoolInterface, ABC):
|
|
|
23
23
|
"""Queues saved items until :meth:`commit`, which the pool implements.
|
|
24
24
|
|
|
25
25
|
:meth:`save` is a deferred save committed at once, so a pool writes in one
|
|
26
|
-
place only. What
|
|
27
|
-
|
|
26
|
+
place only. What was queued before is committed first, apart, so the
|
|
27
|
+
answer is about the saved item alone and not another one's failure. What
|
|
28
|
+
is queued is the pool's to drop when the keys are deleted or cleared, and
|
|
29
|
+
to commit before a queued key is read.
|
|
28
30
|
"""
|
|
29
31
|
|
|
30
32
|
_deferred: dict[str, CacheItem]
|
|
@@ -39,6 +41,9 @@ class DeferredItemsMixin(CacheItemPoolInterface, ABC):
|
|
|
39
41
|
if not isinstance(item, CacheItem):
|
|
40
42
|
return False
|
|
41
43
|
|
|
44
|
+
_ = self._deferred.pop(item.key, None)
|
|
45
|
+
if self._deferred:
|
|
46
|
+
_ = await self.commit()
|
|
42
47
|
self._deferred[item.key] = item
|
|
43
48
|
return await self.commit()
|
|
44
49
|
|
|
@@ -7,6 +7,7 @@ import base64
|
|
|
7
7
|
import hashlib
|
|
8
8
|
import os
|
|
9
9
|
import secrets
|
|
10
|
+
import stat
|
|
10
11
|
import tempfile
|
|
11
12
|
import time
|
|
12
13
|
from pathlib import Path
|
|
@@ -41,6 +42,12 @@ _ABANDONED_AFTER: Final = 3600.0
|
|
|
41
42
|
|
|
42
43
|
_WRITE_FLAGS: Final = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
43
44
|
|
|
45
|
+
_HEADER_BYTES: Final = 4096
|
|
46
|
+
"""Enough of a file to hold its expiry and identifier lines, in all but freak cases."""
|
|
47
|
+
|
|
48
|
+
_PRIVATE_MODE: Final = 0o700
|
|
49
|
+
"""The default directory's mode: in the shared temporary directory, only its owner may enter."""
|
|
50
|
+
|
|
44
51
|
|
|
45
52
|
@final
|
|
46
53
|
class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
@@ -53,10 +60,18 @@ class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
|
53
60
|
|
|
54
61
|
Nothing is created until the first value is written. An expired file is
|
|
55
62
|
removed when it is read; :meth:`prune` removes the ones nobody reads.
|
|
63
|
+
|
|
64
|
+
Without a directory of its own, a pool keeps its files in the system's
|
|
65
|
+
temporary directory, where every user may create one first. Values are
|
|
66
|
+
unpickled when read, so there the directory is made for this user alone,
|
|
67
|
+
and one belonging to anybody else is refused: every read misses and every
|
|
68
|
+
write fails, and the pool logs why.
|
|
56
69
|
"""
|
|
57
70
|
|
|
58
71
|
_directory: Path
|
|
59
72
|
_marshaller: MarshallerInterface
|
|
73
|
+
_private_root: Path | None
|
|
74
|
+
_private_root_checked: bool
|
|
60
75
|
|
|
61
76
|
def __init__(
|
|
62
77
|
self,
|
|
@@ -91,6 +106,8 @@ class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
|
91
106
|
Path(directory) if directory is not None else Path(tempfile.gettempdir()) / "xtr-cache"
|
|
92
107
|
)
|
|
93
108
|
self._directory = base / (namespace or _NO_NAMESPACE)
|
|
109
|
+
self._private_root = base if directory is None else None
|
|
110
|
+
self._private_root_checked = False
|
|
94
111
|
self._marshaller = marshaller if marshaller is not None else DefaultMarshaller()
|
|
95
112
|
|
|
96
113
|
@property
|
|
@@ -138,6 +155,8 @@ class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
|
138
155
|
|
|
139
156
|
def _read_all(self, ids: Sequence[str], now: float) -> dict[str, bytes]:
|
|
140
157
|
found: dict[str, bytes] = {}
|
|
158
|
+
if not self._is_trusted(create=False):
|
|
159
|
+
return found
|
|
141
160
|
for id_ in ids:
|
|
142
161
|
content = self._read(self._path(id_))
|
|
143
162
|
if content is None:
|
|
@@ -153,17 +172,19 @@ class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
|
153
172
|
return found
|
|
154
173
|
|
|
155
174
|
def _is_live(self, id_: str, now: float) -> bool:
|
|
156
|
-
|
|
157
|
-
if content is None or content[1] != id_:
|
|
175
|
+
if not self._is_trusted(create=False):
|
|
158
176
|
return False
|
|
159
|
-
|
|
177
|
+
header = self._read_header(self._path(id_))
|
|
178
|
+
if header is None or header[1] != id_:
|
|
179
|
+
return False
|
|
180
|
+
return not header[0] or header[0] > now
|
|
160
181
|
|
|
161
182
|
def _clear(self, namespace: str) -> bool:
|
|
162
183
|
ok = True
|
|
163
184
|
for path in self._files():
|
|
164
185
|
if namespace:
|
|
165
|
-
|
|
166
|
-
if
|
|
186
|
+
header = self._read_header(path)
|
|
187
|
+
if header is None or not header[1].startswith(namespace):
|
|
167
188
|
continue
|
|
168
189
|
ok = _unlink(path) and ok
|
|
169
190
|
|
|
@@ -176,6 +197,8 @@ class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
|
176
197
|
return ok
|
|
177
198
|
|
|
178
199
|
def _write_all(self, encoded: Mapping[str, bytes], expiry: float) -> list[str]:
|
|
200
|
+
if not self._is_trusted(create=True):
|
|
201
|
+
return list(encoded)
|
|
179
202
|
failed: list[str] = []
|
|
180
203
|
for id_, data in encoded.items():
|
|
181
204
|
path = self._path(id_)
|
|
@@ -196,12 +219,55 @@ class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
|
196
219
|
|
|
197
220
|
return failed
|
|
198
221
|
|
|
222
|
+
def _is_trusted(self, *, create: bool) -> bool:
|
|
223
|
+
"""Tell whether the files under the default directory may be read and written.
|
|
224
|
+
|
|
225
|
+
Only the default directory is checked: one the application chose is
|
|
226
|
+
its own. Creating it makes it private; an existing one of this user's
|
|
227
|
+
is made private too. Once found trustworthy, it stays so: in a
|
|
228
|
+
temporary directory nobody else may rename or remove it.
|
|
229
|
+
"""
|
|
230
|
+
root = self._private_root
|
|
231
|
+
if root is None or self._private_root_checked:
|
|
232
|
+
return True
|
|
233
|
+
try:
|
|
234
|
+
if create:
|
|
235
|
+
root.mkdir(mode=_PRIVATE_MODE, parents=True, exist_ok=True)
|
|
236
|
+
status = root.lstat()
|
|
237
|
+
except FileNotFoundError:
|
|
238
|
+
return False
|
|
239
|
+
except OSError as error:
|
|
240
|
+
self._log(
|
|
241
|
+
'Cannot use the cache directory "{directory}": {reason}', error, directory=root
|
|
242
|
+
)
|
|
243
|
+
return False
|
|
244
|
+
|
|
245
|
+
owner = os.getuid() if hasattr(os, "getuid") else status.st_uid
|
|
246
|
+
if not stat.S_ISDIR(status.st_mode) or status.st_uid != owner:
|
|
247
|
+
self._log(
|
|
248
|
+
'Refusing the cache directory "{directory}": it is not a directory of this user.',
|
|
249
|
+
directory=root,
|
|
250
|
+
)
|
|
251
|
+
return False
|
|
252
|
+
if stat.S_IMODE(status.st_mode) != _PRIVATE_MODE:
|
|
253
|
+
try:
|
|
254
|
+
root.chmod(_PRIVATE_MODE)
|
|
255
|
+
except OSError as error:
|
|
256
|
+
self._log(
|
|
257
|
+
'Cannot make the cache directory "{directory}" private: {reason}',
|
|
258
|
+
error,
|
|
259
|
+
directory=root,
|
|
260
|
+
)
|
|
261
|
+
return False
|
|
262
|
+
self._private_root_checked = True
|
|
263
|
+
return True
|
|
264
|
+
|
|
199
265
|
def _prune(self) -> bool:
|
|
200
266
|
now = self._clock.now().timestamp()
|
|
201
267
|
ok = True
|
|
202
268
|
for path in self._files():
|
|
203
|
-
|
|
204
|
-
if
|
|
269
|
+
header = self._read_header(path)
|
|
270
|
+
if header is not None and header[0] and header[0] <= now:
|
|
205
271
|
ok = _unlink(path) and ok
|
|
206
272
|
|
|
207
273
|
# File times are the file system's, so they are compared with the machine's clock.
|
|
@@ -221,6 +287,29 @@ class FilesystemAdapter(AbstractAdapter, PruneableInterface):
|
|
|
221
287
|
if name.startswith(_TEMPORARY_PREFIX) is temporary:
|
|
222
288
|
yield Path(root) / name
|
|
223
289
|
|
|
290
|
+
@staticmethod
|
|
291
|
+
def _read_header(path: Path) -> tuple[float, str] | None:
|
|
292
|
+
"""Return a file's expiry and identifier, reading no more of it than they take.
|
|
293
|
+
|
|
294
|
+
Pruning and clearing look at every file; reading each value in full
|
|
295
|
+
to see two short lines would read the whole cache.
|
|
296
|
+
"""
|
|
297
|
+
try:
|
|
298
|
+
with path.open("rb") as file:
|
|
299
|
+
head = file.read(_HEADER_BYTES)
|
|
300
|
+
except OSError:
|
|
301
|
+
return None
|
|
302
|
+
if head.count(b"\n") < 2: # noqa: PLR2004 — the expiry line and the identifier line
|
|
303
|
+
# An identifier longer than the bound: read the file as a whole.
|
|
304
|
+
content = FilesystemAdapter._read(path)
|
|
305
|
+
return None if content is None else (content[0], content[1])
|
|
306
|
+
expiry, _, rest = head.partition(b"\n")
|
|
307
|
+
stored_id, _, _ = rest.partition(b"\n")
|
|
308
|
+
try:
|
|
309
|
+
return float(expiry), unquote(stored_id.decode("ascii"))
|
|
310
|
+
except (UnicodeDecodeError, ValueError):
|
|
311
|
+
return None
|
|
312
|
+
|
|
224
313
|
@staticmethod
|
|
225
314
|
def _read(path: Path) -> tuple[float, str, bytes] | None:
|
|
226
315
|
"""Return a file's expiry, identifier and value bytes; ``None`` when missing or not ours."""
|
|
@@ -28,6 +28,10 @@ if TYPE_CHECKING:
|
|
|
28
28
|
__all__ = ["TagAwareAdapter"]
|
|
29
29
|
|
|
30
30
|
TAGS_PREFIX: Final = "\x01tags\x01"
|
|
31
|
+
|
|
32
|
+
#: How long a tag's version is kept. Given explicitly, so a tags pool's default
|
|
33
|
+
#: lifetime never drops a version before the values that were saved with it.
|
|
34
|
+
_TAG_VERSION_LIFETIME: Final = 100 * 365 * 86400
|
|
31
35
|
"""What the key holding a tag's version starts with, so it never meets a user key."""
|
|
32
36
|
|
|
33
37
|
|
|
@@ -244,9 +248,8 @@ class TagAwareAdapter(
|
|
|
244
248
|
created = [tag for tag, found in current.items() if found is None]
|
|
245
249
|
|
|
246
250
|
for tag in created:
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
)
|
|
251
|
+
version = CacheItem(TAGS_PREFIX + tag, versions[tag], clock=self._clock)
|
|
252
|
+
_ = await self._tags.save_deferred(version.expires_after(_TAG_VERSION_LIFETIME))
|
|
250
253
|
# A version the tags pool did not keep must not be trusted here while others miss it.
|
|
251
254
|
if created and await self._tags.commit():
|
|
252
255
|
now = self._clock.now().timestamp()
|
|
@@ -58,8 +58,7 @@ class CacheConfig:
|
|
|
58
58
|
app: The ``app`` pool's adapter, or several to chain, fastest first.
|
|
59
59
|
pools: Every other pool, by name.
|
|
60
60
|
directory: Where ``"filesystem"`` keeps its files: ``cache`` under the
|
|
61
|
-
kernel's ``share_dir`` by default,
|
|
62
|
-
temporary one set aside for this project.
|
|
61
|
+
kernel's ``share_dir`` by default, ``var/share`` in the project.
|
|
63
62
|
prefix_seed: What every pool's namespace is derived from, with its
|
|
64
63
|
name: the project directory by default, so two applications
|
|
65
64
|
sharing a backend never see each other's keys. Give two
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import asyncio
|
|
6
|
+
import time
|
|
6
7
|
import zlib
|
|
7
8
|
from typing import TYPE_CHECKING, Final, TypeVar, cast, final
|
|
8
9
|
|
|
@@ -41,16 +42,17 @@ class LockRegistry(LoggerAware):
|
|
|
41
42
|
|
|
42
43
|
A caller that waited longer than ``wait`` stops waiting and computes the
|
|
43
44
|
value itself: a stuck holder must not stall every reader of the key. The
|
|
44
|
-
slot is then
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
slot is then set aside, in this process, for another ``wait``: its keys
|
|
46
|
+
are computed without locking meanwhile rather than wait on it in turn,
|
|
47
|
+
and it is tried again after. A key keeps its slot whatever is set aside,
|
|
48
|
+
so every process agrees on which lock guards it.
|
|
47
49
|
"""
|
|
48
50
|
|
|
49
|
-
__slots__ = ("
|
|
51
|
+
__slots__ = ("_evicted", "_locks", "_prefix", "_slots", "_wait")
|
|
50
52
|
|
|
51
53
|
_locks: LockFactory
|
|
52
54
|
_slots: int
|
|
53
|
-
|
|
55
|
+
_evicted: dict[int, float]
|
|
54
56
|
_wait: float
|
|
55
57
|
_prefix: str
|
|
56
58
|
|
|
@@ -80,7 +82,7 @@ class LockRegistry(LoggerAware):
|
|
|
80
82
|
|
|
81
83
|
self._locks = locks
|
|
82
84
|
self._slots = slots
|
|
83
|
-
self.
|
|
85
|
+
self._evicted = {}
|
|
84
86
|
self._wait = wait
|
|
85
87
|
self._prefix = prefix
|
|
86
88
|
|
|
@@ -140,15 +142,19 @@ class LockRegistry(LoggerAware):
|
|
|
140
142
|
self.logger.info('Item "{key}" not found while lock was released, retrying', context)
|
|
141
143
|
|
|
142
144
|
def _slot(self, key: str) -> int | None:
|
|
143
|
-
"""Return the slot ``key`` computes under; ``None``
|
|
144
|
-
|
|
145
|
+
"""Return the slot ``key`` computes under; ``None`` while that slot is set aside."""
|
|
146
|
+
slot = zlib.crc32(key.encode()) % self._slots
|
|
147
|
+
until = self._evicted.get(slot)
|
|
148
|
+
if until is None:
|
|
149
|
+
return slot
|
|
150
|
+
if time.monotonic() < until:
|
|
145
151
|
return None
|
|
146
|
-
|
|
152
|
+
del self._evicted[slot]
|
|
153
|
+
return slot
|
|
147
154
|
|
|
148
155
|
def _evict(self, slot: int) -> None:
|
|
149
|
-
"""
|
|
150
|
-
|
|
151
|
-
self._available.remove(slot)
|
|
156
|
+
"""Set ``slot`` aside for a while, its holder stuck, so later callers do not wait on it."""
|
|
157
|
+
self._evicted[slot] = time.monotonic() + self._wait
|
|
152
158
|
|
|
153
159
|
@property
|
|
154
160
|
def slots(self) -> int:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -189,10 +189,10 @@ class AbstractAdapter(
|
|
|
189
189
|
|
|
190
190
|
@override
|
|
191
191
|
async def clear(self, prefix: str = "") -> bool:
|
|
192
|
-
self._drop_deferred(prefix)
|
|
193
192
|
if not _PREFIX_PATTERN.fullmatch(prefix):
|
|
194
193
|
self._log("Failed to clear the cache: the prefix contains invalid characters.")
|
|
195
194
|
return False
|
|
195
|
+
self._drop_deferred(prefix)
|
|
196
196
|
|
|
197
197
|
try:
|
|
198
198
|
return await self._do_clear(self._namespace + prefix)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xtr_cache-1.2.0 → xtr_cache-1.4.0}/src/xtr_cache/command/cache_pool_invalidate_tags_command.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|