combycode-dpe 2.0.0rc3__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/PKG-INFO +1 -1
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/pyproject.toml +1 -1
- combycode_dpe-2.0.1/src/dpe/_cache.py +81 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_context.py +52 -4
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_envelope.py +27 -4
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_runtime.py +19 -3
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/pyproject.toml +1 -1
- combycode_dpe-2.0.1/tests/test_cache.py +175 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/tests/test_stderr_events_conformance.py +3 -1
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/.gitignore +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/README.md +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/__init__.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_accumulators.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/meta.json +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/spec.yaml +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/src/__PKG__/__init__.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/src/__PKG__/main.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/tests/test_basic.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/verify/case-basic/expected.ndjson +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/verify/case-basic/input.ndjson +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/verify/case-basic/settings.json +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/tests/test_accumulators.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/tests/test_integration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: combycode-dpe
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.1
|
|
4
4
|
Summary: DPE framework for building streaming pipeline tools in Python
|
|
5
5
|
Project-URL: Repository, https://github.com/combycode/dpe-framework-python
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Cache helper used by Context.cached(). Mirrors the TS framework's
|
|
2
|
+
cache.ts behavior — same DPE_CACHE_MODE semantics, same on-disk layout
|
|
3
|
+
($DPE_STORAGE/<namespace>/<hash>.json), same canonical-JSON key
|
|
4
|
+
hashing.
|
|
5
|
+
|
|
6
|
+
Failure modes (cache-disabling, NOT errors propagated to user):
|
|
7
|
+
- DPE_STORAGE not set -> cache disabled, every call produces
|
|
8
|
+
- cache file unreadable -> treat as miss, log warn
|
|
9
|
+
- cache file unparseable -> treat as miss, log warn
|
|
10
|
+
- producer raises -> re-raise to caller (no cache write)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from typing import Any, TypeVar
|
|
20
|
+
|
|
21
|
+
from dpe._envelope import write_log
|
|
22
|
+
|
|
23
|
+
T = TypeVar("T")
|
|
24
|
+
|
|
25
|
+
CACHE_MODES = ("use", "refresh", "bypass", "off")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def read_cache_mode() -> str:
|
|
29
|
+
"""Read DPE_CACHE_MODE; default to "use". Unrecognized -> "use"."""
|
|
30
|
+
v = os.getenv("DPE_CACHE_MODE", "use")
|
|
31
|
+
return v if v in CACHE_MODES else "use"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def cache_path(namespace: str, key: Any) -> str | None:
|
|
35
|
+
"""Compute the on-disk path for (namespace, key). Returns None when
|
|
36
|
+
DPE_STORAGE isn't set -- caller treats that as cache disabled.
|
|
37
|
+
"""
|
|
38
|
+
storage = os.getenv("DPE_STORAGE")
|
|
39
|
+
if not storage:
|
|
40
|
+
return None
|
|
41
|
+
canonical = json.dumps(key, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
42
|
+
h = hashlib.blake2b(canonical.encode("utf-8")).hexdigest()[:32]
|
|
43
|
+
return os.path.join(storage, namespace, f"{h}.json")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def cached_impl(
|
|
47
|
+
namespace: str,
|
|
48
|
+
key: Any,
|
|
49
|
+
produce: Callable[[], T],
|
|
50
|
+
) -> T:
|
|
51
|
+
"""Read-or-produce cache around `produce`. Honors the four cache
|
|
52
|
+
modes. Returns the produced or cached value."""
|
|
53
|
+
mode = read_cache_mode()
|
|
54
|
+
path = cache_path(namespace, key)
|
|
55
|
+
|
|
56
|
+
can_read = path is not None and mode in ("use", "refresh")
|
|
57
|
+
can_write = path is not None and mode not in ("bypass", "off")
|
|
58
|
+
will_read = can_read and mode != "refresh"
|
|
59
|
+
|
|
60
|
+
if will_read and path is not None and os.path.isfile(path):
|
|
61
|
+
try:
|
|
62
|
+
with open(path, encoding="utf-8") as f:
|
|
63
|
+
value = json.load(f)
|
|
64
|
+
write_log(f"cached: hit ({namespace})", "debug")
|
|
65
|
+
return value
|
|
66
|
+
except (OSError, json.JSONDecodeError) as e:
|
|
67
|
+
write_log(f"cached: read failed ({namespace}) — {e}", "warn")
|
|
68
|
+
# Fall through to produce.
|
|
69
|
+
|
|
70
|
+
result = produce()
|
|
71
|
+
|
|
72
|
+
if can_write and path is not None:
|
|
73
|
+
try:
|
|
74
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
75
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
76
|
+
json.dump(result, f, ensure_ascii=False, separators=(",", ":"))
|
|
77
|
+
except OSError as e:
|
|
78
|
+
write_log(f"cached: write failed ({namespace}) — {e}", "warn")
|
|
79
|
+
# Don't fail the caller — they got their value.
|
|
80
|
+
|
|
81
|
+
return result
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
"""Context object — passed to every processor invocation."""
|
|
2
2
|
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any, TypeVar
|
|
5
|
+
|
|
3
6
|
from dpe._accumulators import Memory
|
|
7
|
+
from dpe._cache import cached_impl
|
|
4
8
|
from dpe._envelope import (
|
|
5
9
|
hash_file,
|
|
6
10
|
hash_string,
|
|
@@ -12,6 +16,8 @@ from dpe._envelope import (
|
|
|
12
16
|
write_trace,
|
|
13
17
|
)
|
|
14
18
|
|
|
19
|
+
T = TypeVar("T")
|
|
20
|
+
|
|
15
21
|
|
|
16
22
|
class Context:
|
|
17
23
|
"""Processing context created per invocation.
|
|
@@ -50,13 +56,14 @@ class Context:
|
|
|
50
56
|
def output(self, v, *, id: str | None = None, src: str | None = None):
|
|
51
57
|
"""Emit data record to stdout.
|
|
52
58
|
|
|
53
|
-
Flushes accumulated trace labels as a merged trace event first
|
|
54
|
-
|
|
59
|
+
Flushes accumulated trace labels as a merged trace event first
|
|
60
|
+
(channel="data" → counts as rows_out at the runner), then writes
|
|
61
|
+
the envelope, then clears the label bag.
|
|
55
62
|
"""
|
|
56
63
|
out_id = id if id is not None else self.id
|
|
57
64
|
out_src = src if src is not None else self.src
|
|
58
65
|
# Emit trace even with empty labels — the chain row itself is the value.
|
|
59
|
-
write_trace(out_id, out_src, self._labels)
|
|
66
|
+
write_trace(out_id, out_src, self._labels, channel="data")
|
|
60
67
|
self._labels = {}
|
|
61
68
|
write_data(v, out_id, out_src)
|
|
62
69
|
|
|
@@ -82,7 +89,13 @@ class Context:
|
|
|
82
89
|
self._runtime.drain_queue()
|
|
83
90
|
|
|
84
91
|
def meta(self, v: dict):
|
|
85
|
-
"""Emit metadata record to stdout.
|
|
92
|
+
"""Emit metadata record to stdout.
|
|
93
|
+
|
|
94
|
+
Also emits a {type:"trace", channel:"meta"} stderr event so the
|
|
95
|
+
runner can increment its per-stage `meta` counter. Inherits
|
|
96
|
+
ctx id/src.
|
|
97
|
+
"""
|
|
98
|
+
write_trace(self.id, self.src, {}, channel="meta")
|
|
86
99
|
write_meta(v)
|
|
87
100
|
|
|
88
101
|
def log(self, msg: str, *, level: str = "info", **extra):
|
|
@@ -100,3 +113,38 @@ class Context:
|
|
|
100
113
|
def hash_file(self, filepath: str, algorithm: str = "blake2b") -> str | None:
|
|
101
114
|
"""Hash file content in chunks. Returns hex string or None on error."""
|
|
102
115
|
return hash_file(filepath, algorithm)
|
|
116
|
+
|
|
117
|
+
def cached(
|
|
118
|
+
self,
|
|
119
|
+
namespace: str,
|
|
120
|
+
key: Any,
|
|
121
|
+
produce: Callable[[], T],
|
|
122
|
+
) -> T:
|
|
123
|
+
"""Cache the result of `produce()` under
|
|
124
|
+
`$DPE_STORAGE/<namespace>/<hash>.json`.
|
|
125
|
+
|
|
126
|
+
Honors the runner's `DPE_CACHE_MODE` env:
|
|
127
|
+
- use (default) -- read cache if present, else produce + write
|
|
128
|
+
- refresh -- always produce, overwrite cache
|
|
129
|
+
- bypass -- produce, skip both read and write
|
|
130
|
+
- off -- same as bypass
|
|
131
|
+
|
|
132
|
+
`key` is canonical-JSON-hashed (blake2b, first 32 hex chars).
|
|
133
|
+
Compose it from whatever determines output equivalence.
|
|
134
|
+
|
|
135
|
+
If `$DPE_STORAGE` isn't set (e.g. tool invoked outside a
|
|
136
|
+
pipeline), cache is silently disabled — every call produces.
|
|
137
|
+
|
|
138
|
+
Producer errors propagate; failed runs do NOT poison the cache.
|
|
139
|
+
|
|
140
|
+
Example:
|
|
141
|
+
result = ctx.cached(
|
|
142
|
+
"doc-converter",
|
|
143
|
+
{"file_hash": ctx.hash_file(v["path"]),
|
|
144
|
+
"settings": settings,
|
|
145
|
+
"page": page_idx},
|
|
146
|
+
lambda: provider.convert_page(...),
|
|
147
|
+
)
|
|
148
|
+
ctx.output(result)
|
|
149
|
+
"""
|
|
150
|
+
return cached_impl(namespace, key, produce)
|
|
@@ -26,7 +26,12 @@ def hash_file(filepath: str, algorithm: str = "blake2b", chunk_size: int = 65536
|
|
|
26
26
|
try:
|
|
27
27
|
if algorithm == "xxhash":
|
|
28
28
|
try:
|
|
29
|
-
|
|
29
|
+
# xxhash is an optional runtime dep (see fallback). Need
|
|
30
|
+
# both codes here: `import-not-found` fires when xxhash isn't
|
|
31
|
+
# installed (CI), `unused-ignore` fires when it IS installed
|
|
32
|
+
# (some dev machines). Listing both keeps mypy happy in both
|
|
33
|
+
# environments without disabling the warning globally.
|
|
34
|
+
import xxhash # type: ignore[import-not-found,unused-ignore]
|
|
30
35
|
h = xxhash.xxh3_64()
|
|
31
36
|
except ImportError:
|
|
32
37
|
h = hashlib.blake2b(digest_size=16)
|
|
@@ -89,9 +94,27 @@ def write_error(v, err, id: str, src: str):
|
|
|
89
94
|
_write_stderr(record)
|
|
90
95
|
|
|
91
96
|
|
|
92
|
-
def write_trace(id: str, src: str, labels: dict):
|
|
93
|
-
"""Write merged trace event to stderr. Called by ctx before each output.
|
|
94
|
-
|
|
97
|
+
def write_trace(id: str, src: str, labels: dict, channel: str | None = None):
|
|
98
|
+
"""Write merged trace event to stderr. Called by ctx before each output.
|
|
99
|
+
|
|
100
|
+
`channel` distinguishes counter buckets at the runner side:
|
|
101
|
+
- "data" from ctx.output -> rows_out
|
|
102
|
+
- "meta" from ctx.meta -> meta
|
|
103
|
+
None is written without a `channel` key (back-compat: runner treats
|
|
104
|
+
unset as data).
|
|
105
|
+
"""
|
|
106
|
+
record: dict = {"type": "trace", "id": id, "src": src, "labels": labels}
|
|
107
|
+
if channel is not None:
|
|
108
|
+
record["channel"] = channel
|
|
109
|
+
_write_stderr(record)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def write_input(id: str, src: str):
|
|
113
|
+
"""Write input event to stderr. Emitted by the runtime per envelope read
|
|
114
|
+
from stdin, BEFORE the processor runs. Lets the runner count rows_in
|
|
115
|
+
for stages that never call ctx.output() (pass-through, terminal sinks).
|
|
116
|
+
"""
|
|
117
|
+
_write_stderr({"type": "input", "id": id, "src": src})
|
|
95
118
|
|
|
96
119
|
|
|
97
120
|
def write_stats(data: dict):
|
|
@@ -9,7 +9,7 @@ from collections.abc import Callable
|
|
|
9
9
|
|
|
10
10
|
from dpe._accumulators import Memory
|
|
11
11
|
from dpe._context import Context
|
|
12
|
-
from dpe._envelope import parse_envelope, write_log
|
|
12
|
+
from dpe._envelope import parse_envelope, write_input, write_log
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class Runtime:
|
|
@@ -71,6 +71,14 @@ class Runtime:
|
|
|
71
71
|
|
|
72
72
|
# Parse settings
|
|
73
73
|
self.settings = _parse_settings()
|
|
74
|
+
# `accept_meta: true` in settings → the read loop dispatches t:"m"
|
|
75
|
+
# envelopes to process_input alongside t:"d" (instead of skipping).
|
|
76
|
+
# Default false preserves historical behavior — meta envelopes only
|
|
77
|
+
# matter to tools that explicitly opt in (typically sinks like
|
|
78
|
+
# write-file-stream when used as a meta-output target).
|
|
79
|
+
accept_meta = bool(
|
|
80
|
+
isinstance(self.settings, dict) and self.settings.get("accept_meta") is True
|
|
81
|
+
)
|
|
74
82
|
|
|
75
83
|
# Main stdin loop
|
|
76
84
|
for line in sys.stdin:
|
|
@@ -81,14 +89,22 @@ class Runtime:
|
|
|
81
89
|
if envelope is None:
|
|
82
90
|
continue
|
|
83
91
|
|
|
84
|
-
#
|
|
85
|
-
|
|
92
|
+
# Default: only data envelopes. With accept_meta=True, also
|
|
93
|
+
# accept t:"m". Anything else (unknown / missing t) skipped.
|
|
94
|
+
t = envelope.get("t")
|
|
95
|
+
if t != "d" and not (accept_meta and t == "m"):
|
|
86
96
|
continue
|
|
87
97
|
|
|
88
98
|
id = envelope.get("id", "")
|
|
89
99
|
src = envelope.get("src", "")
|
|
90
100
|
v = envelope.get("v", {})
|
|
91
101
|
|
|
102
|
+
# Emit an `input` event BEFORE the processor runs so the runner
|
|
103
|
+
# can count rows_in for every stage that reads stdin — including
|
|
104
|
+
# pass-through tools and terminal sinks that never call
|
|
105
|
+
# ctx.output() (and therefore wouldn't otherwise emit a trace).
|
|
106
|
+
write_input(id, src)
|
|
107
|
+
|
|
92
108
|
ctx = self.create_ctx(id, src)
|
|
93
109
|
try:
|
|
94
110
|
self.input_fn(v, self.settings, ctx)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Tests for the cache helper. Mirror coverage of the TS suite."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import tempfile
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
11
|
+
from dpe._cache import cache_path, cached_impl, read_cache_mode
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@pytest.fixture
|
|
15
|
+
def storage_dir(monkeypatch):
|
|
16
|
+
"""Set DPE_STORAGE to a tempdir for the test, clear DPE_CACHE_MODE."""
|
|
17
|
+
with tempfile.TemporaryDirectory(prefix="dpe-cache-test-") as d:
|
|
18
|
+
monkeypatch.setenv("DPE_STORAGE", d)
|
|
19
|
+
monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
|
|
20
|
+
yield d
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ─── read_cache_mode ─────────────────────────────────────────────────────
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_read_cache_mode_default(monkeypatch):
|
|
27
|
+
monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
|
|
28
|
+
assert read_cache_mode() == "use"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize("m", ["use", "refresh", "bypass", "off"])
|
|
32
|
+
def test_read_cache_mode_recognized(monkeypatch, m):
|
|
33
|
+
monkeypatch.setenv("DPE_CACHE_MODE", m)
|
|
34
|
+
assert read_cache_mode() == m
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_read_cache_mode_garbage(monkeypatch):
|
|
38
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "nonsense")
|
|
39
|
+
assert read_cache_mode() == "use"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ─── cache_path ──────────────────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_cache_path_none_when_no_storage(monkeypatch):
|
|
46
|
+
monkeypatch.delenv("DPE_STORAGE", raising=False)
|
|
47
|
+
assert cache_path("ns", {"k": 1}) is None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_cache_path_key_order_stable(storage_dir):
|
|
51
|
+
a = cache_path("ns", {"k": 1, "m": 2})
|
|
52
|
+
b = cache_path("ns", {"m": 2, "k": 1})
|
|
53
|
+
assert a == b
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_cache_path_different_keys(storage_dir):
|
|
57
|
+
assert cache_path("ns", {"k": 1}) != cache_path("ns", {"k": 2})
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_cache_path_different_namespaces(storage_dir):
|
|
61
|
+
assert cache_path("ns1", {"k": 1}) != cache_path("ns2", {"k": 1})
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ─── cached_impl ─────────────────────────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_miss_calls_produce_writes_file(storage_dir):
|
|
68
|
+
calls = [0]
|
|
69
|
+
|
|
70
|
+
def produce():
|
|
71
|
+
calls[0] += 1
|
|
72
|
+
return {"hello": "world"}
|
|
73
|
+
|
|
74
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
75
|
+
assert result == {"hello": "world"}
|
|
76
|
+
assert calls[0] == 1
|
|
77
|
+
assert Path(cache_path("ns", {"k": 1})).is_file()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_hit_skips_produce(storage_dir):
|
|
81
|
+
# Prime via miss to create the dir.
|
|
82
|
+
cached_impl("ns", {"k": 1}, lambda: {"initial": True})
|
|
83
|
+
# Overwrite with our seeded value.
|
|
84
|
+
Path(cache_path("ns", {"k": 1})).write_text(json.dumps({"cached": True}))
|
|
85
|
+
|
|
86
|
+
calls = [0]
|
|
87
|
+
|
|
88
|
+
def produce():
|
|
89
|
+
calls[0] += 1
|
|
90
|
+
return {"fresh": True}
|
|
91
|
+
|
|
92
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
93
|
+
assert result == {"cached": True}
|
|
94
|
+
assert calls[0] == 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_refresh_always_produces_overwrites(storage_dir, monkeypatch):
|
|
98
|
+
cached_impl("ns", {"k": 1}, lambda: {"first": True})
|
|
99
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "refresh")
|
|
100
|
+
|
|
101
|
+
calls = [0]
|
|
102
|
+
|
|
103
|
+
def produce():
|
|
104
|
+
calls[0] += 1
|
|
105
|
+
return {"fresh": True}
|
|
106
|
+
|
|
107
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
108
|
+
assert result == {"fresh": True}
|
|
109
|
+
assert calls[0] == 1
|
|
110
|
+
on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
|
|
111
|
+
assert on_disk == {"fresh": True}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_bypass_no_read_no_write(storage_dir, monkeypatch):
|
|
115
|
+
cached_impl("ns", {"k": 1}, lambda: {"initial": True})
|
|
116
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "bypass")
|
|
117
|
+
|
|
118
|
+
result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
|
|
119
|
+
assert result == {"fresh": True}
|
|
120
|
+
on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
|
|
121
|
+
# bypass does NOT write — original cache content unchanged.
|
|
122
|
+
assert on_disk == {"initial": True}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_off_same_as_bypass(storage_dir, monkeypatch):
|
|
126
|
+
cached_impl("ns", {"k": 1}, lambda: {"initial": True})
|
|
127
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "off")
|
|
128
|
+
result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
|
|
129
|
+
assert result == {"fresh": True}
|
|
130
|
+
on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
|
|
131
|
+
assert on_disk == {"initial": True}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_no_storage_disables_cache(monkeypatch):
|
|
135
|
+
monkeypatch.delenv("DPE_STORAGE", raising=False)
|
|
136
|
+
monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
|
|
137
|
+
|
|
138
|
+
calls = [0]
|
|
139
|
+
|
|
140
|
+
def produce():
|
|
141
|
+
calls[0] += 1
|
|
142
|
+
return {"n": calls[0]}
|
|
143
|
+
|
|
144
|
+
r1 = cached_impl("ns", {"k": 1}, produce)
|
|
145
|
+
r2 = cached_impl("ns", {"k": 1}, produce)
|
|
146
|
+
assert r1 == {"n": 1}
|
|
147
|
+
assert r2 == {"n": 2}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_producer_error_propagates_no_write(storage_dir):
|
|
151
|
+
attempts = [0]
|
|
152
|
+
|
|
153
|
+
def boom():
|
|
154
|
+
attempts[0] += 1
|
|
155
|
+
raise RuntimeError("kaboom")
|
|
156
|
+
|
|
157
|
+
with pytest.raises(RuntimeError, match="kaboom"):
|
|
158
|
+
cached_impl("ns", {"k": 1}, boom)
|
|
159
|
+
assert attempts[0] == 1
|
|
160
|
+
assert not Path(cache_path("ns", {"k": 1})).is_file()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_malformed_cache_treated_as_miss(storage_dir):
|
|
164
|
+
cached_impl("ns", {"k": 1}, lambda: {"ok": True})
|
|
165
|
+
Path(cache_path("ns", {"k": 1})).write_text("this is not json")
|
|
166
|
+
|
|
167
|
+
calls = [0]
|
|
168
|
+
|
|
169
|
+
def produce():
|
|
170
|
+
calls[0] += 1
|
|
171
|
+
return {"recovered": True}
|
|
172
|
+
|
|
173
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
174
|
+
assert result == {"recovered": True}
|
|
175
|
+
assert calls[0] == 1
|
|
@@ -20,7 +20,9 @@ import pytest
|
|
|
20
20
|
|
|
21
21
|
from dpe import _envelope as env
|
|
22
22
|
|
|
23
|
-
SCHEMA_PATH =
|
|
23
|
+
SCHEMA_PATH = (
|
|
24
|
+
Path(__file__).resolve().parents[3] / "runner" / "schemas" / "stderr-events.schema.json"
|
|
25
|
+
)
|
|
24
26
|
|
|
25
27
|
|
|
26
28
|
@pytest.fixture(scope="module")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|