dialcache 0.25.0__tar.gz → 0.27.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dialcache-0.25.0 → dialcache-0.27.0}/API-DESIGN.md +14 -3
- {dialcache-0.25.0 → dialcache-0.27.0}/PKG-INFO +28 -11
- {dialcache-0.25.0 → dialcache-0.27.0}/README.md +26 -10
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/__init__.py +3 -1
- dialcache-0.27.0/dialcache/_callbacks.py +27 -0
- dialcache-0.27.0/dialcache/_shadow_log.py +324 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/cache.py +164 -65
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/config.py +120 -12
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/key.py +11 -1
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/serializer.py +13 -5
- {dialcache-0.25.0 → dialcache-0.27.0}/pyproject.toml +2 -2
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/driver.py +63 -14
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/scenarios.py +1 -1
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/witness.py +9 -9
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/run_conformance.py +2 -2
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/run_integration.py +11 -2
- dialcache-0.27.0/tests/test_cached_keys.py +123 -0
- dialcache-0.27.0/tests/test_config.py +404 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_conformance.py +2 -2
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_docs_examples.py +34 -3
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_integration_acceptance.py +47 -22
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_observer_recovery_boundaries.py +85 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_protocol_vectors.py +36 -7
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_redis_integration.py +5 -2
- dialcache-0.27.0/tests/test_shadow_log.py +826 -0
- dialcache-0.27.0/tests/test_shadow_mismatch_logging.py +797 -0
- dialcache-0.27.0/tests/test_typing.py +33 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_validation_environment.py +1 -1
- dialcache-0.27.0/tests/test_wire_suite.py +38 -0
- dialcache-0.27.0/tests/typing/cached.py +115 -0
- dialcache-0.25.0/tests/test_config.py +0 -142
- {dialcache-0.25.0 → dialcache-0.27.0}/.gitignore +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/LICENSE +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/clock.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/context.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/errors.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/local.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/metrics.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/protocol.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/py.typed +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/dialcache/redis.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/__init__.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/coordinator.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/executor.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/schema.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/formal/simple_drivers.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_cache.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_callback_cancellation.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_cluster_read_routing.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_context.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_dependency_cancellation.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_engine_review.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_local.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_protocol_native.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_redis_adapter.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_review_regressions.py +0 -0
- {dialcache-0.25.0 → dialcache-0.27.0}/tests/test_shadow_deadline_retention.py +0 -0
|
@@ -2,13 +2,24 @@
|
|
|
2
2
|
|
|
3
3
|
The Python binding was designed after reviewing [rungalileo/gcache at a688049](https://github.com/rungalileo/gcache/tree/a68804986782cb0b7e8b7dc6bfc34c718c177189), including `src/gcache/gcache.py`, `config.py`, `proto_serializer.py`, and the context, layer wrappers, local cache, Redis cache, and event-loop thread implementations under `_internal/`.
|
|
4
4
|
|
|
5
|
-
DialCache's [Quint specification](../formal/SPEC.md) and TypeScript implementation define portable behavior. Gcache supplies useful Python interface ideas; its implementation is not a compatible DialCache backend.
|
|
5
|
+
DialCache's [Quint specification](../formal/guides/SPEC.md) and TypeScript implementation define portable behavior. Gcache supplies useful Python interface ideas; its implementation is not a compatible DialCache backend.
|
|
6
|
+
|
|
7
|
+
The primary decorator style selects identity explicitly with
|
|
8
|
+
`cache_key=lambda user_id, locale: {"id": user_id, "args": {"locale": locale}}`
|
|
9
|
+
and a stable `use_case`. This follows TypeScript's `cacheKey` while retaining
|
|
10
|
+
Python decorator syntax. The selector makes result dimensions and their wire
|
|
11
|
+
names explicit; it must include every input that affects the result.
|
|
12
|
+
|
|
13
|
+
Selectors receive the loader's original `*args` and `**kwargs`. Match parameter
|
|
14
|
+
names and defaults so positional, keyword, and omitted-default calls select
|
|
15
|
+
the intended identity. Loader defaults are not injected into the selector;
|
|
16
|
+
changing that behavior could change existing keys or break existing callbacks.
|
|
6
17
|
|
|
7
18
|
| Gcache interface or behavior | Python DialCache decision |
|
|
8
19
|
| --- | --- |
|
|
9
20
|
| `with cache.enable(enabled=True)` | Retained, with per-instance context variables and explicit outer-scope lifetime. Also supports `async with` and `disable()`. |
|
|
10
|
-
| `@cache.cached(key_type=..., id_arg=...)` |
|
|
11
|
-
| `arg_adapters`, `ignore_args`, inferred use case |
|
|
21
|
+
| `@cache.cached(key_type=..., id_arg=...)` | Supported alternative to explicit `cache_key` selection. `id_arg` can be a parameter name or `(name, adapter)` pair. Signature binding includes defaults. |
|
|
22
|
+
| `arg_adapters`, `ignore_args`, inferred use case | Inferred argument controls remain supported. Default use-case names include module and qualified function name; prefer explicit names for stability across refactors and languages. Argument names use the portable UTF-16 ordering and scalar normalization. |
|
|
12
23
|
| Direct `aget(key, fallback)` | Retained as a structured-key convenience; `get_or_load` is the primary inline-loader API. |
|
|
13
24
|
| `ainvalidate` | Retained as an alias for `invalidate_remote`; missing Redis and failed mutations raise. |
|
|
14
25
|
| `GCacheKeyConfig` and per-use-case provider | `Policy` / `DialCacheKeyConfig` use sparse per-leaf inheritance and deterministic per-key cohorts. `Policy.enabled(ttl_sec)` is available. |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: dialcache
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.27.0
|
|
4
4
|
Summary: Explicitly enabled async caching with runtime policies, coalescing, and tracked Redis invalidation.
|
|
5
5
|
Project-URL: Repository, https://github.com/lan17/DialCache
|
|
6
6
|
Project-URL: Documentation, https://github.com/lan17/DialCache/tree/main/python
|
|
@@ -21,6 +21,7 @@ Requires-Dist: redis<7,>=5; extra == 'redis'
|
|
|
21
21
|
Provides-Extra: test
|
|
22
22
|
Requires-Dist: coverage<8,>=7.11; extra == 'test'
|
|
23
23
|
Requires-Dist: jsonschema<5,>=4.23; extra == 'test'
|
|
24
|
+
Requires-Dist: mypy<2,>=1.15; extra == 'test'
|
|
24
25
|
Requires-Dist: pytest-asyncio<2,>=0.24; extra == 'test'
|
|
25
26
|
Requires-Dist: pytest<10,>=8; extra == 'test'
|
|
26
27
|
Description-Content-Type: text/markdown
|
|
@@ -30,7 +31,7 @@ Description-Content-Type: text/markdown
|
|
|
30
31
|
An asyncio port of DialCache for Python 3.11 and later. Each use case declares
|
|
31
32
|
its identity and policy; an enabled request scope opts into request memoization,
|
|
32
33
|
local storage, Redis, and concurrent request sharing. The behavioral contract is
|
|
33
|
-
the repository's [portable specification](https://github.com/lan17/DialCache/blob/main/formal/SPEC.md).
|
|
34
|
+
the repository's [portable specification](https://github.com/lan17/DialCache/blob/main/formal/guides/SPEC.md).
|
|
34
35
|
|
|
35
36
|
Once the first PyPI release is available, install it with:
|
|
36
37
|
|
|
@@ -50,6 +51,9 @@ any Redis client connections.
|
|
|
50
51
|
|
|
51
52
|
## A cached function
|
|
52
53
|
|
|
54
|
+
Select cache identity explicitly with `cache_key`. This API excerpt assumes an
|
|
55
|
+
application `database`:
|
|
56
|
+
|
|
53
57
|
```python
|
|
54
58
|
from dialcache import DialCache, Policy
|
|
55
59
|
|
|
@@ -59,20 +63,33 @@ cache = DialCache(namespace="my-service")
|
|
|
59
63
|
@cache.cached(
|
|
60
64
|
use_case="user-profile",
|
|
61
65
|
key_type="user",
|
|
62
|
-
|
|
66
|
+
cache_key=lambda user_id, locale: {"id": user_id, "args": {"locale": locale}},
|
|
63
67
|
default_config=Policy(ttl_sec={"local": 5}, request_local=True),
|
|
64
68
|
)
|
|
65
|
-
async def get_profile(user_id: str) -> dict:
|
|
66
|
-
return await database.fetch_profile(user_id)
|
|
69
|
+
async def get_profile(user_id: str, locale: str) -> dict:
|
|
70
|
+
return await database.fetch_profile(user_id, locale)
|
|
67
71
|
|
|
68
72
|
|
|
69
|
-
async def handle_request(user_id: str) -> dict:
|
|
73
|
+
async def handle_request(user_id: str, locale: str) -> dict:
|
|
70
74
|
async with cache.enable():
|
|
71
|
-
first = await get_profile(user_id)
|
|
72
|
-
second = await get_profile(user_id) # Same request memo.
|
|
75
|
+
first = await get_profile(user_id, locale)
|
|
76
|
+
second = await get_profile(user_id, locale) # Same request memo.
|
|
73
77
|
return second
|
|
74
78
|
```
|
|
75
79
|
|
|
80
|
+
The selector returns an entity ID, or `{"id": ..., "args": {...}}` when the
|
|
81
|
+
result has additional dimensions. Here, `id` groups the user's tracked values
|
|
82
|
+
for invalidation; `args` distinguishes locales within this use case. Include
|
|
83
|
+
every input that affects the result, including tenant or authorization scope
|
|
84
|
+
when relevant. Explicit `use_case` and argument names keep identity stable
|
|
85
|
+
across function refactors and languages.
|
|
86
|
+
|
|
87
|
+
The selector receives the same positional and keyword arguments as the loader.
|
|
88
|
+
Match their parameter names and repeat any defaults the loader accepts; loader
|
|
89
|
+
defaults are not filled in before calling the selector. For inferred identity,
|
|
90
|
+
`id_arg`, `arg_adapters`, and `ignore_args` remain supported alternatives; see
|
|
91
|
+
the [Python identity guide](https://lan17.github.io/DialCache/languages/python.html#identity-and-policy).
|
|
92
|
+
|
|
76
93
|
Calls outside `enable()` go directly to the source. They do not construct cache
|
|
77
94
|
keys, resolve policy, share concurrent work, or apply a DialCache source
|
|
78
95
|
deadline. Both `with cache.enable():` and `async with cache.enable():` are valid;
|
|
@@ -157,7 +174,7 @@ cache = DialCache(redis=RedisAdapter(client))
|
|
|
157
174
|
@cache.cached(
|
|
158
175
|
use_case="user-profile",
|
|
159
176
|
key_type="user",
|
|
160
|
-
|
|
177
|
+
cache_key=lambda user_id: user_id,
|
|
161
178
|
track_for_invalidation=True,
|
|
162
179
|
default_config=Policy(ttl_sec={"remote": 60}),
|
|
163
180
|
)
|
|
@@ -251,6 +268,6 @@ local expiry, cancellation, and wire boundaries. Shared replay runs the real
|
|
|
251
268
|
Python API through the repository's Node coordinator. Its inputs and expected
|
|
252
269
|
observations come from the same Quint-generated histories used by the other
|
|
253
270
|
ports; Node is a development dependency, not a runtime dependency of the
|
|
254
|
-
Python library. See [the porting guide](https://github.com/lan17/DialCache/blob/main/formal/PORTING.md) for the completion
|
|
255
|
-
and settlement requirements and [the feature map](https://github.com/lan17/DialCache/blob/main/formal/FEATURE-COVERAGE.md)
|
|
271
|
+
Python library. See [the porting guide](https://github.com/lan17/DialCache/blob/main/formal/guides/PORTING.md) for the completion
|
|
272
|
+
and settlement requirements and [the feature map](https://github.com/lan17/DialCache/blob/main/formal/guides/FEATURE-COVERAGE.md)
|
|
256
273
|
for portable behavior versus native adapter obligations.
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
An asyncio port of DialCache for Python 3.11 and later. Each use case declares
|
|
4
4
|
its identity and policy; an enabled request scope opts into request memoization,
|
|
5
5
|
local storage, Redis, and concurrent request sharing. The behavioral contract is
|
|
6
|
-
the repository's [portable specification](https://github.com/lan17/DialCache/blob/main/formal/SPEC.md).
|
|
6
|
+
the repository's [portable specification](https://github.com/lan17/DialCache/blob/main/formal/guides/SPEC.md).
|
|
7
7
|
|
|
8
8
|
Once the first PyPI release is available, install it with:
|
|
9
9
|
|
|
@@ -23,6 +23,9 @@ any Redis client connections.
|
|
|
23
23
|
|
|
24
24
|
## A cached function
|
|
25
25
|
|
|
26
|
+
Select cache identity explicitly with `cache_key`. This API excerpt assumes an
|
|
27
|
+
application `database`:
|
|
28
|
+
|
|
26
29
|
```python
|
|
27
30
|
from dialcache import DialCache, Policy
|
|
28
31
|
|
|
@@ -32,20 +35,33 @@ cache = DialCache(namespace="my-service")
|
|
|
32
35
|
@cache.cached(
|
|
33
36
|
use_case="user-profile",
|
|
34
37
|
key_type="user",
|
|
35
|
-
|
|
38
|
+
cache_key=lambda user_id, locale: {"id": user_id, "args": {"locale": locale}},
|
|
36
39
|
default_config=Policy(ttl_sec={"local": 5}, request_local=True),
|
|
37
40
|
)
|
|
38
|
-
async def get_profile(user_id: str) -> dict:
|
|
39
|
-
return await database.fetch_profile(user_id)
|
|
41
|
+
async def get_profile(user_id: str, locale: str) -> dict:
|
|
42
|
+
return await database.fetch_profile(user_id, locale)
|
|
40
43
|
|
|
41
44
|
|
|
42
|
-
async def handle_request(user_id: str) -> dict:
|
|
45
|
+
async def handle_request(user_id: str, locale: str) -> dict:
|
|
43
46
|
async with cache.enable():
|
|
44
|
-
first = await get_profile(user_id)
|
|
45
|
-
second = await get_profile(user_id) # Same request memo.
|
|
47
|
+
first = await get_profile(user_id, locale)
|
|
48
|
+
second = await get_profile(user_id, locale) # Same request memo.
|
|
46
49
|
return second
|
|
47
50
|
```
|
|
48
51
|
|
|
52
|
+
The selector returns an entity ID, or `{"id": ..., "args": {...}}` when the
|
|
53
|
+
result has additional dimensions. Here, `id` groups the user's tracked values
|
|
54
|
+
for invalidation; `args` distinguishes locales within this use case. Include
|
|
55
|
+
every input that affects the result, including tenant or authorization scope
|
|
56
|
+
when relevant. Explicit `use_case` and argument names keep identity stable
|
|
57
|
+
across function refactors and languages.
|
|
58
|
+
|
|
59
|
+
The selector receives the same positional and keyword arguments as the loader.
|
|
60
|
+
Match their parameter names and repeat any defaults the loader accepts; loader
|
|
61
|
+
defaults are not filled in before calling the selector. For inferred identity,
|
|
62
|
+
`id_arg`, `arg_adapters`, and `ignore_args` remain supported alternatives; see
|
|
63
|
+
the [Python identity guide](https://lan17.github.io/DialCache/languages/python.html#identity-and-policy).
|
|
64
|
+
|
|
49
65
|
Calls outside `enable()` go directly to the source. They do not construct cache
|
|
50
66
|
keys, resolve policy, share concurrent work, or apply a DialCache source
|
|
51
67
|
deadline. Both `with cache.enable():` and `async with cache.enable():` are valid;
|
|
@@ -130,7 +146,7 @@ cache = DialCache(redis=RedisAdapter(client))
|
|
|
130
146
|
@cache.cached(
|
|
131
147
|
use_case="user-profile",
|
|
132
148
|
key_type="user",
|
|
133
|
-
|
|
149
|
+
cache_key=lambda user_id: user_id,
|
|
134
150
|
track_for_invalidation=True,
|
|
135
151
|
default_config=Policy(ttl_sec={"remote": 60}),
|
|
136
152
|
)
|
|
@@ -224,6 +240,6 @@ local expiry, cancellation, and wire boundaries. Shared replay runs the real
|
|
|
224
240
|
Python API through the repository's Node coordinator. Its inputs and expected
|
|
225
241
|
observations come from the same Quint-generated histories used by the other
|
|
226
242
|
ports; Node is a development dependency, not a runtime dependency of the
|
|
227
|
-
Python library. See [the porting guide](https://github.com/lan17/DialCache/blob/main/formal/PORTING.md) for the completion
|
|
228
|
-
and settlement requirements and [the feature map](https://github.com/lan17/DialCache/blob/main/formal/FEATURE-COVERAGE.md)
|
|
243
|
+
Python library. See [the porting guide](https://github.com/lan17/DialCache/blob/main/formal/guides/PORTING.md) for the completion
|
|
244
|
+
and settlement requirements and [the feature map](https://github.com/lan17/DialCache/blob/main/formal/guides/FEATURE-COVERAGE.md)
|
|
229
245
|
for portable behavior versus native adapter obligations.
|
|
@@ -12,7 +12,7 @@ from .errors import (
|
|
|
12
12
|
UseCaseIsAlreadyRegisteredError,
|
|
13
13
|
UseCaseNameIsReservedError,
|
|
14
14
|
)
|
|
15
|
-
from .key import Key, normalize_args
|
|
15
|
+
from .key import CacheKeySpec, Key, KeyScalar, normalize_args
|
|
16
16
|
from .serializer import UNDEFINED, JsonSerializer, Serializer
|
|
17
17
|
|
|
18
18
|
__all__ = [
|
|
@@ -22,6 +22,8 @@ __all__ = [
|
|
|
22
22
|
"DialCacheKeyConfig",
|
|
23
23
|
"CacheLayer",
|
|
24
24
|
"Key",
|
|
25
|
+
"CacheKeySpec",
|
|
26
|
+
"KeyScalar",
|
|
25
27
|
"normalize_args",
|
|
26
28
|
"Serializer",
|
|
27
29
|
"JsonSerializer",
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Isolation for application callbacks whose contract is synchronous."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import contextvars
|
|
7
|
+
import inspect
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def discard_awaitable(value: Any) -> None:
|
|
12
|
+
"""Fail closed on an awaitable a synchronous callback returned.
|
|
13
|
+
|
|
14
|
+
A rejected result must not start work or take ownership of an
|
|
15
|
+
application's existing task: a coroutine that has not started is closed,
|
|
16
|
+
and a future's exception is consumed without awaiting it.
|
|
17
|
+
"""
|
|
18
|
+
if inspect.iscoroutine(value):
|
|
19
|
+
if inspect.getcoroutinestate(value) == inspect.CORO_CREATED:
|
|
20
|
+
value.close()
|
|
21
|
+
elif asyncio.isfuture(value):
|
|
22
|
+
|
|
23
|
+
def consume(done: asyncio.Future[Any]) -> None:
|
|
24
|
+
if not done.cancelled():
|
|
25
|
+
done.exception()
|
|
26
|
+
|
|
27
|
+
value.add_done_callback(consume, context=contextvars.Context())
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""Bounded content of confirmed shadow-mismatch warnings.
|
|
2
|
+
|
|
3
|
+
Every field derives from one native JSON rendering of each compared value, so
|
|
4
|
+
a failure logs less, never more. The built-in difference computation walks the
|
|
5
|
+
two parsed renderings and reports RFC 6901 JSON Pointers.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import inspect
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
from collections.abc import Callable
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from ._callbacks import discard_awaitable
|
|
19
|
+
from .protocol import utf8_bytes
|
|
20
|
+
from .serializer import UNDEFINED, _native_json
|
|
21
|
+
|
|
22
|
+
KEY_MAX_BYTES = 2 * 1024
|
|
23
|
+
VALUE_MAX_BYTES = 8 * 1024
|
|
24
|
+
DIFF_MAX_BYTES = 8 * 1024
|
|
25
|
+
MAX_ENTRIES = 64
|
|
26
|
+
PATH_MAX_BYTES = 512
|
|
27
|
+
SIDE_MAX_BYTES = 1024
|
|
28
|
+
# Reference tokens in the deepest pointer the built-in walk reports.
|
|
29
|
+
MAX_DEPTH = 32
|
|
30
|
+
TRUNCATION_MARKER = "...[truncated]"
|
|
31
|
+
_MARKER_BYTES = TRUNCATION_MARKER.encode()
|
|
32
|
+
|
|
33
|
+
# One difference: a pointer, and each side under "cached" or "source" when the
|
|
34
|
+
# value has that side. A missing key is an absent side; None is JSON null.
|
|
35
|
+
Entry = dict[str, Any]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class MismatchLogPlan:
|
|
40
|
+
"""The effective content leaves captured when a shadow job is admitted."""
|
|
41
|
+
|
|
42
|
+
value: bool
|
|
43
|
+
diff: str
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def clamp_utf8(text: str, limit: int) -> str:
|
|
47
|
+
"""Clamp to ``limit`` UTF-8 bytes with the marker, never splitting a code point."""
|
|
48
|
+
encoded = utf8_bytes(text)
|
|
49
|
+
if len(encoded) <= limit:
|
|
50
|
+
return text
|
|
51
|
+
return encoded[: limit - len(_MARKER_BYTES)].decode("utf-8", "ignore") + TRUNCATION_MARKER
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def render_json(value: Any) -> str | None:
|
|
55
|
+
"""The native JSON rendering ``JsonSerializer`` stores, or None for a value without one.
|
|
56
|
+
|
|
57
|
+
Lone surrogates are escaped, so the rendering is valid UTF-8. A top-level
|
|
58
|
+
``UNDEFINED`` has none here: its stored sentinel is not JSON.
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
return _native_json(value)
|
|
62
|
+
except Exception:
|
|
63
|
+
return None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def render_shadow_mismatch_log(
|
|
67
|
+
plan: MismatchLogPlan,
|
|
68
|
+
cached: Any,
|
|
69
|
+
source: Any,
|
|
70
|
+
hook: Callable[[Any, Any], Any] | None = None,
|
|
71
|
+
) -> tuple[dict[str, Any], str | None]:
|
|
72
|
+
"""Build the content fields the plan requests for one confirmed mismatch.
|
|
73
|
+
|
|
74
|
+
Returns the fields, each requested one present and None when it failed
|
|
75
|
+
closed, and the failure: ``"hook_error"`` for the hook, which ranks above
|
|
76
|
+
``"render_error"`` for JSON rendering, or None. Each side is rendered at
|
|
77
|
+
most once. A failed hook never falls back to the built-in walk. A
|
|
78
|
+
top-level ``UNDEFINED``, which a negative-cache entry stores, is an absent
|
|
79
|
+
side rather than a failure: its value field is None, and the walk reports
|
|
80
|
+
the other side whole at ``""``.
|
|
81
|
+
"""
|
|
82
|
+
render_sides = plan.value or (plan.diff != "off" and hook is None)
|
|
83
|
+
cached_json = render_json(cached) if render_sides else None
|
|
84
|
+
source_json = render_json(source) if render_sides else None
|
|
85
|
+
diff_failed = False
|
|
86
|
+
render_failed = render_sides and (
|
|
87
|
+
(cached_json is None and cached is not UNDEFINED) or (source_json is None and source is not UNDEFINED)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
fields: dict[str, Any] = {}
|
|
91
|
+
if plan.diff != "off":
|
|
92
|
+
with_values = plan.diff == "values"
|
|
93
|
+
diff: tuple[list[Entry], int] | None = None
|
|
94
|
+
if hook is not None:
|
|
95
|
+
diff = _run_hook(hook, cached, source)
|
|
96
|
+
diff_failed = diff is None
|
|
97
|
+
elif not render_failed:
|
|
98
|
+
diff = _walk_rendered(cached_json, source_json)
|
|
99
|
+
if diff is None:
|
|
100
|
+
fields["mismatchPaths"] = None
|
|
101
|
+
fields["mismatchPathCount"] = None
|
|
102
|
+
if with_values:
|
|
103
|
+
fields["diffJson"] = None
|
|
104
|
+
else:
|
|
105
|
+
entries, count = diff
|
|
106
|
+
diff_json = None
|
|
107
|
+
if with_values:
|
|
108
|
+
diff_json = _render_diff_json(entries)
|
|
109
|
+
render_failed = render_failed or diff_json is None
|
|
110
|
+
# The paths level never renders a side.
|
|
111
|
+
fields["mismatchPaths"] = [clamp_utf8(entry["path"], PATH_MAX_BYTES) for entry in entries]
|
|
112
|
+
fields["mismatchPathCount"] = count
|
|
113
|
+
if with_values:
|
|
114
|
+
fields["diffJson"] = diff_json
|
|
115
|
+
|
|
116
|
+
if plan.value:
|
|
117
|
+
fields["cachedValueJson"] = None if cached_json is None else clamp_utf8(cached_json, VALUE_MAX_BYTES)
|
|
118
|
+
fields["sourceValueJson"] = None if source_json is None else clamp_utf8(source_json, VALUE_MAX_BYTES)
|
|
119
|
+
return fields, "hook_error" if diff_failed else "render_error" if render_failed else None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def diff_shadow_mismatch_json(cached: Any, source: Any) -> tuple[list[Entry | None], int]:
|
|
123
|
+
"""The built-in difference computation over two JSON values.
|
|
124
|
+
|
|
125
|
+
Objects recurse over the union of their member names in UTF-16 code-unit
|
|
126
|
+
order, and a member on one side only is reported with that side. Arrays of
|
|
127
|
+
equal length recurse by index; arrays of different length are reported once
|
|
128
|
+
at their own pointer. Any other differing pair, roots of different kinds
|
|
129
|
+
included, is reported with both sides, and a differing pair of containers
|
|
130
|
+
at ``MAX_DEPTH`` reference tokens is reported whole. Entries keep walk
|
|
131
|
+
order and are cut at ``MAX_ENTRIES``; the count is the uncut total. Each
|
|
132
|
+
pointer and side is clamped as in ``diffJson``.
|
|
133
|
+
"""
|
|
134
|
+
entries, count = _walk(cached, source)
|
|
135
|
+
# JSON values always render, so every entry bounds.
|
|
136
|
+
return [_bound_entry(entry) for entry in entries], count
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _walk_rendered(cached_json: str | None, source_json: str | None) -> tuple[list[Entry], int]:
|
|
140
|
+
# None is an absent side; a side that failed to render never reaches here.
|
|
141
|
+
# Each string came from a successful native rendering, so this is the one
|
|
142
|
+
# boundary from arbitrary values to the closed JSON domain.
|
|
143
|
+
if cached_json is None:
|
|
144
|
+
return ([], 0) if source_json is None else ([{"path": "", "source": json.loads(source_json)}], 1)
|
|
145
|
+
if source_json is None:
|
|
146
|
+
return [{"path": "", "cached": json.loads(cached_json)}], 1
|
|
147
|
+
if cached_json == source_json:
|
|
148
|
+
return [], 0
|
|
149
|
+
return _walk(json.loads(cached_json), json.loads(source_json))
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _walk(cached: Any, source: Any) -> tuple[list[Entry], int]:
|
|
153
|
+
entries: list[Entry] = []
|
|
154
|
+
count = 0
|
|
155
|
+
# The escaped reference tokens down to the pair being walked. Python
|
|
156
|
+
# strings have no ropes, so a pointer is built only for a retained entry:
|
|
157
|
+
# copying the prefix for each member would cost members times its length.
|
|
158
|
+
tokens: list[str] = []
|
|
159
|
+
|
|
160
|
+
def report(**sides: Any) -> None:
|
|
161
|
+
nonlocal count
|
|
162
|
+
count += 1
|
|
163
|
+
if len(entries) < MAX_ENTRIES:
|
|
164
|
+
entries.append({"path": _clamped_pointer(tokens), **sides})
|
|
165
|
+
|
|
166
|
+
def walk(left: Any, right: Any) -> None:
|
|
167
|
+
kind = _kind(left)
|
|
168
|
+
if kind != _kind(right) or kind == "primitive":
|
|
169
|
+
if not _same_primitive(left, right):
|
|
170
|
+
report(cached=left, source=right)
|
|
171
|
+
return
|
|
172
|
+
if len(tokens) >= MAX_DEPTH:
|
|
173
|
+
if not _json_equal(left, right):
|
|
174
|
+
report(cached=left, source=right)
|
|
175
|
+
return
|
|
176
|
+
if kind == "array":
|
|
177
|
+
if len(left) != len(right):
|
|
178
|
+
report(cached=left, source=right)
|
|
179
|
+
return
|
|
180
|
+
for index, (item, other) in enumerate(zip(left, right)):
|
|
181
|
+
tokens.append(str(index))
|
|
182
|
+
walk(item, other)
|
|
183
|
+
tokens.pop()
|
|
184
|
+
return
|
|
185
|
+
for name in sorted(left.keys() | right.keys(), key=_utf16_order):
|
|
186
|
+
tokens.append(_escape_token(name))
|
|
187
|
+
if name not in right:
|
|
188
|
+
report(cached=left[name])
|
|
189
|
+
elif name not in left:
|
|
190
|
+
report(source=right[name])
|
|
191
|
+
else:
|
|
192
|
+
walk(left[name], right[name])
|
|
193
|
+
tokens.pop()
|
|
194
|
+
|
|
195
|
+
walk(cached, source)
|
|
196
|
+
return entries, count
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _clamped_pointer(tokens: list[str]) -> str:
|
|
200
|
+
# A token longer than the cap in code points is longer than it in UTF-8
|
|
201
|
+
# bytes, so cutting each token there keeps every byte the clamp keeps.
|
|
202
|
+
return clamp_utf8("".join(f"/{token[:PATH_MAX_BYTES]}" for token in tokens), PATH_MAX_BYTES)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _json_equal(cached: Any, source: Any) -> bool:
|
|
206
|
+
# Structural equality for a depth-capped pair, iterative so that a subtree
|
|
207
|
+
# as deep as native JSON rendering allows cannot exhaust the stack.
|
|
208
|
+
pending = [(cached, source)]
|
|
209
|
+
while pending:
|
|
210
|
+
left, right = pending.pop()
|
|
211
|
+
kind = _kind(left)
|
|
212
|
+
if kind != _kind(right):
|
|
213
|
+
return False
|
|
214
|
+
if kind == "primitive":
|
|
215
|
+
if not _same_primitive(left, right):
|
|
216
|
+
return False
|
|
217
|
+
elif kind == "array":
|
|
218
|
+
if len(left) != len(right):
|
|
219
|
+
return False
|
|
220
|
+
pending.extend(zip(left, right))
|
|
221
|
+
else:
|
|
222
|
+
if len(left) != len(right):
|
|
223
|
+
return False
|
|
224
|
+
for name, value in left.items():
|
|
225
|
+
if name not in right:
|
|
226
|
+
return False
|
|
227
|
+
pending.append((value, right[name]))
|
|
228
|
+
return True
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _kind(value: Any) -> str:
|
|
232
|
+
if isinstance(value, list):
|
|
233
|
+
return "array"
|
|
234
|
+
return "object" if isinstance(value, dict) else "primitive"
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _same_primitive(left: Any, right: Any) -> bool:
|
|
238
|
+
# JSON equality, not Python's: True == 1 and False == 0 in Python.
|
|
239
|
+
if isinstance(left, bool) or isinstance(right, bool) or left is None or right is None:
|
|
240
|
+
return left is right
|
|
241
|
+
if isinstance(left, str) or isinstance(right, str):
|
|
242
|
+
return isinstance(left, str) and isinstance(right, str) and left == right
|
|
243
|
+
if isinstance(left, (int, float)) and isinstance(right, (int, float)):
|
|
244
|
+
# Numbers compare as IEEE doubles, as the other ports parse them, so
|
|
245
|
+
# precision beyond a double is not a difference.
|
|
246
|
+
return _as_double(left) == _as_double(right)
|
|
247
|
+
return False
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _as_double(number: int | float) -> float:
|
|
251
|
+
# float() rounds an int to nearest and raises exactly when IEEE rounding
|
|
252
|
+
# gives an infinity, the value TypeScript parses from the rendered digits.
|
|
253
|
+
# math.copysign would raise too: it converts the int to a float first.
|
|
254
|
+
try:
|
|
255
|
+
return float(number)
|
|
256
|
+
except OverflowError:
|
|
257
|
+
return math.inf if number > 0 else -math.inf
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _utf16_order(name: str) -> bytes:
|
|
261
|
+
# Big-endian code units compare as UTF-16 code units, the W02 key order.
|
|
262
|
+
return name.encode("utf-16-be", "surrogatepass")
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _escape_token(name: str) -> str:
|
|
266
|
+
return name.replace("~", "~0").replace("/", "~1")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _run_hook(hook: Callable[[Any, Any], Any], cached: Any, source: Any) -> tuple[list[Entry], int] | None:
|
|
270
|
+
"""Run the use case's hook under the comparator's synchronous discipline.
|
|
271
|
+
|
|
272
|
+
A raise, an awaitable or a result that is not a list returns None. Only
|
|
273
|
+
the kept members are read, inside the try, and a path is logged as its
|
|
274
|
+
str form: a hook that returns wrong entries gets wrong paths.
|
|
275
|
+
"""
|
|
276
|
+
try:
|
|
277
|
+
result = hook(cached, source)
|
|
278
|
+
if inspect.isawaitable(result):
|
|
279
|
+
# The contract is synchronous: fail closed without driving it.
|
|
280
|
+
discard_awaitable(result)
|
|
281
|
+
return None
|
|
282
|
+
if not isinstance(result, list):
|
|
283
|
+
return None
|
|
284
|
+
entries: list[Entry] = []
|
|
285
|
+
for member in result[:MAX_ENTRIES]:
|
|
286
|
+
entry: Entry = {"path": str(member["path"])}
|
|
287
|
+
for side in ("cached", "source"):
|
|
288
|
+
if side in member and member[side] is not UNDEFINED:
|
|
289
|
+
entry[side] = member[side]
|
|
290
|
+
entries.append(entry)
|
|
291
|
+
return entries, len(result)
|
|
292
|
+
except (Exception, asyncio.CancelledError):
|
|
293
|
+
return None
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _render_diff_json(entries: list[Entry]) -> str | None:
|
|
297
|
+
# A side with no JSON rendering makes the whole field None, and later
|
|
298
|
+
# sides are not rendered.
|
|
299
|
+
bounded = []
|
|
300
|
+
for entry in entries:
|
|
301
|
+
next_entry = _bound_entry(entry)
|
|
302
|
+
if next_entry is None:
|
|
303
|
+
return None
|
|
304
|
+
bounded.append(next_entry)
|
|
305
|
+
return clamp_utf8(_native_json(bounded), DIFF_MAX_BYTES)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _bound_entry(entry: Entry) -> Entry | None:
|
|
309
|
+
"""Clamp the pointer, and keep each present side as JSON or as its clamped text.
|
|
310
|
+
|
|
311
|
+
One clamp rule for built-in and hook entries. None when a side has no JSON
|
|
312
|
+
rendering.
|
|
313
|
+
"""
|
|
314
|
+
bounded: Entry = {"path": clamp_utf8(entry["path"], PATH_MAX_BYTES)}
|
|
315
|
+
for side in ("cached", "source"):
|
|
316
|
+
if side not in entry:
|
|
317
|
+
continue
|
|
318
|
+
text = render_json(entry[side])
|
|
319
|
+
if text is None:
|
|
320
|
+
return None
|
|
321
|
+
clamped = clamp_utf8(text, SIDE_MAX_BYTES)
|
|
322
|
+
# Parsing the one rendering keeps each side rendered once.
|
|
323
|
+
bounded[side] = json.loads(text) if clamped is text else clamped
|
|
324
|
+
return bounded
|