combycode-dpe 2.0.0rc3__tar.gz → 2.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/PKG-INFO +1 -1
  2. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/pyproject.toml +1 -1
  3. combycode_dpe-2.0.1/src/dpe/_cache.py +81 -0
  4. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_context.py +52 -4
  5. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_envelope.py +27 -4
  6. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_runtime.py +19 -3
  7. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/pyproject.toml +1 -1
  8. combycode_dpe-2.0.1/tests/test_cache.py +175 -0
  9. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/tests/test_stderr_events_conformance.py +3 -1
  10. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/.gitignore +0 -0
  11. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/README.md +0 -0
  12. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/__init__.py +0 -0
  13. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/src/dpe/_accumulators.py +0 -0
  14. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/meta.json +0 -0
  15. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/spec.yaml +0 -0
  16. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/src/__PKG__/__init__.py +0 -0
  17. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/src/__PKG__/main.py +0 -0
  18. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/tests/test_basic.py +0 -0
  19. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/verify/case-basic/expected.ndjson +0 -0
  20. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/verify/case-basic/input.ndjson +0 -0
  21. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/template/verify/case-basic/settings.json +0 -0
  22. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/tests/test_accumulators.py +0 -0
  23. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.1}/tests/test_integration.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: combycode-dpe
3
- Version: 2.0.0rc3
3
+ Version: 2.0.1
4
4
  Summary: DPE framework for building streaming pipeline tools in Python
5
5
  Project-URL: Repository, https://github.com/combycode/dpe-framework-python
6
6
  License-Expression: AGPL-3.0-or-later
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "combycode-dpe"
7
- version = "2.0.0rc3"
7
+ version = "2.0.1"
8
8
  description = "DPE framework for building streaming pipeline tools in Python"
9
9
  license = "AGPL-3.0-or-later"
10
10
  requires-python = ">=3.11"
@@ -0,0 +1,81 @@
1
+ """Cache helper used by Context.cached(). Mirrors the TS framework's
2
+ cache.ts behavior — same DPE_CACHE_MODE semantics, same on-disk layout
3
+ ($DPE_STORAGE/<namespace>/<hash>.json), same canonical-JSON key
4
+ hashing.
5
+
6
+ Failure modes (cache-disabling, NOT errors propagated to user):
7
+ - DPE_STORAGE not set -> cache disabled, every call produces
8
+ - cache file unreadable -> treat as miss, log warn
9
+ - cache file unparseable -> treat as miss, log warn
10
+ - producer raises -> re-raise to caller (no cache write)
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import hashlib
16
+ import json
17
+ import os
18
+ from collections.abc import Callable
19
+ from typing import Any, TypeVar
20
+
21
+ from dpe._envelope import write_log
22
+
23
+ T = TypeVar("T")
24
+
25
+ CACHE_MODES = ("use", "refresh", "bypass", "off")
26
+
27
+
28
+ def read_cache_mode() -> str:
29
+ """Read DPE_CACHE_MODE; default to "use". Unrecognized -> "use"."""
30
+ v = os.getenv("DPE_CACHE_MODE", "use")
31
+ return v if v in CACHE_MODES else "use"
32
+
33
+
34
+ def cache_path(namespace: str, key: Any) -> str | None:
35
+ """Compute the on-disk path for (namespace, key). Returns None when
36
+ DPE_STORAGE isn't set -- caller treats that as cache disabled.
37
+ """
38
+ storage = os.getenv("DPE_STORAGE")
39
+ if not storage:
40
+ return None
41
+ canonical = json.dumps(key, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
42
+ h = hashlib.blake2b(canonical.encode("utf-8")).hexdigest()[:32]
43
+ return os.path.join(storage, namespace, f"{h}.json")
44
+
45
+
46
+ def cached_impl(
47
+ namespace: str,
48
+ key: Any,
49
+ produce: Callable[[], T],
50
+ ) -> T:
51
+ """Read-or-produce cache around `produce`. Honors the four cache
52
+ modes. Returns the produced or cached value."""
53
+ mode = read_cache_mode()
54
+ path = cache_path(namespace, key)
55
+
56
+ can_read = path is not None and mode in ("use", "refresh")
57
+ can_write = path is not None and mode not in ("bypass", "off")
58
+ will_read = can_read and mode != "refresh"
59
+
60
+ if will_read and path is not None and os.path.isfile(path):
61
+ try:
62
+ with open(path, encoding="utf-8") as f:
63
+ value = json.load(f)
64
+ write_log(f"cached: hit ({namespace})", "debug")
65
+ return value
66
+ except (OSError, json.JSONDecodeError) as e:
67
+ write_log(f"cached: read failed ({namespace}) — {e}", "warn")
68
+ # Fall through to produce.
69
+
70
+ result = produce()
71
+
72
+ if can_write and path is not None:
73
+ try:
74
+ os.makedirs(os.path.dirname(path), exist_ok=True)
75
+ with open(path, "w", encoding="utf-8") as f:
76
+ json.dump(result, f, ensure_ascii=False, separators=(",", ":"))
77
+ except OSError as e:
78
+ write_log(f"cached: write failed ({namespace}) — {e}", "warn")
79
+ # Don't fail the caller — they got their value.
80
+
81
+ return result
@@ -1,6 +1,10 @@
1
1
  """Context object — passed to every processor invocation."""
2
2
 
3
+ from collections.abc import Callable
4
+ from typing import Any, TypeVar
5
+
3
6
  from dpe._accumulators import Memory
7
+ from dpe._cache import cached_impl
4
8
  from dpe._envelope import (
5
9
  hash_file,
6
10
  hash_string,
@@ -12,6 +16,8 @@ from dpe._envelope import (
12
16
  write_trace,
13
17
  )
14
18
 
19
+ T = TypeVar("T")
20
+
15
21
 
16
22
  class Context:
17
23
  """Processing context created per invocation.
@@ -50,13 +56,14 @@ class Context:
50
56
  def output(self, v, *, id: str | None = None, src: str | None = None):
51
57
  """Emit data record to stdout.
52
58
 
53
- Flushes accumulated trace labels as a merged trace event first,
54
- then writes the envelope, then clears the label bag.
59
+ Flushes accumulated trace labels as a merged trace event first
60
+ (channel="data" → counts as rows_out at the runner), then writes
61
+ the envelope, then clears the label bag.
55
62
  """
56
63
  out_id = id if id is not None else self.id
57
64
  out_src = src if src is not None else self.src
58
65
  # Emit trace even with empty labels — the chain row itself is the value.
59
- write_trace(out_id, out_src, self._labels)
66
+ write_trace(out_id, out_src, self._labels, channel="data")
60
67
  self._labels = {}
61
68
  write_data(v, out_id, out_src)
62
69
 
@@ -82,7 +89,13 @@ class Context:
82
89
  self._runtime.drain_queue()
83
90
 
84
91
  def meta(self, v: dict):
85
- """Emit metadata record to stdout."""
92
+ """Emit metadata record to stdout.
93
+
94
+ Also emits a {type:"trace", channel:"meta"} stderr event so the
95
+ runner can increment its per-stage `meta` counter. Inherits
96
+ ctx id/src.
97
+ """
98
+ write_trace(self.id, self.src, {}, channel="meta")
86
99
  write_meta(v)
87
100
 
88
101
  def log(self, msg: str, *, level: str = "info", **extra):
@@ -100,3 +113,38 @@ class Context:
100
113
  def hash_file(self, filepath: str, algorithm: str = "blake2b") -> str | None:
101
114
  """Hash file content in chunks. Returns hex string or None on error."""
102
115
  return hash_file(filepath, algorithm)
116
+
117
+ def cached(
118
+ self,
119
+ namespace: str,
120
+ key: Any,
121
+ produce: Callable[[], T],
122
+ ) -> T:
123
+ """Cache the result of `produce()` under
124
+ `$DPE_STORAGE/<namespace>/<hash>.json`.
125
+
126
+ Honors the runner's `DPE_CACHE_MODE` env:
127
+ - use (default) -- read cache if present, else produce + write
128
+ - refresh -- always produce, overwrite cache
129
+ - bypass -- produce, skip both read and write
130
+ - off -- same as bypass
131
+
132
+ `key` is canonical-JSON-hashed (blake2b, first 32 hex chars).
133
+ Compose it from whatever determines output equivalence.
134
+
135
+ If `$DPE_STORAGE` isn't set (e.g. tool invoked outside a
136
+ pipeline), cache is silently disabled — every call produces.
137
+
138
+ Producer errors propagate; failed runs do NOT poison the cache.
139
+
140
+ Example:
141
+ result = ctx.cached(
142
+ "doc-converter",
143
+ {"file_hash": ctx.hash_file(v["path"]),
144
+ "settings": settings,
145
+ "page": page_idx},
146
+ lambda: provider.convert_page(...),
147
+ )
148
+ ctx.output(result)
149
+ """
150
+ return cached_impl(namespace, key, produce)
@@ -26,7 +26,12 @@ def hash_file(filepath: str, algorithm: str = "blake2b", chunk_size: int = 65536
26
26
  try:
27
27
  if algorithm == "xxhash":
28
28
  try:
29
- import xxhash # type: ignore[import-not-found]
29
+ # xxhash is an optional runtime dep (see fallback). Need
30
+ # both codes here: `import-not-found` fires when xxhash isn't
31
+ # installed (CI), `unused-ignore` fires when it IS installed
32
+ # (some dev machines). Listing both keeps mypy happy in both
33
+ # environments without disabling the warning globally.
34
+ import xxhash # type: ignore[import-not-found,unused-ignore]
30
35
  h = xxhash.xxh3_64()
31
36
  except ImportError:
32
37
  h = hashlib.blake2b(digest_size=16)
@@ -89,9 +94,27 @@ def write_error(v, err, id: str, src: str):
89
94
  _write_stderr(record)
90
95
 
91
96
 
92
- def write_trace(id: str, src: str, labels: dict):
93
- """Write merged trace event to stderr. Called by ctx before each output."""
94
- _write_stderr({"type": "trace", "id": id, "src": src, "labels": labels})
97
+ def write_trace(id: str, src: str, labels: dict, channel: str | None = None):
98
+ """Write merged trace event to stderr. Called by ctx before each output.
99
+
100
+ `channel` distinguishes counter buckets at the runner side:
101
+ - "data" from ctx.output -> rows_out
102
+ - "meta" from ctx.meta -> meta
103
+ None is written without a `channel` key (back-compat: runner treats
104
+ unset as data).
105
+ """
106
+ record: dict = {"type": "trace", "id": id, "src": src, "labels": labels}
107
+ if channel is not None:
108
+ record["channel"] = channel
109
+ _write_stderr(record)
110
+
111
+
112
+ def write_input(id: str, src: str):
113
+ """Write input event to stderr. Emitted by the runtime per envelope read
114
+ from stdin, BEFORE the processor runs. Lets the runner count rows_in
115
+ for stages that never call ctx.output() (pass-through, terminal sinks).
116
+ """
117
+ _write_stderr({"type": "input", "id": id, "src": src})
95
118
 
96
119
 
97
120
  def write_stats(data: dict):
@@ -9,7 +9,7 @@ from collections.abc import Callable
9
9
 
10
10
  from dpe._accumulators import Memory
11
11
  from dpe._context import Context
12
- from dpe._envelope import parse_envelope, write_log
12
+ from dpe._envelope import parse_envelope, write_input, write_log
13
13
 
14
14
 
15
15
  class Runtime:
@@ -71,6 +71,14 @@ class Runtime:
71
71
 
72
72
  # Parse settings
73
73
  self.settings = _parse_settings()
74
+ # `accept_meta: true` in settings → the read loop dispatches t:"m"
75
+ # envelopes to process_input alongside t:"d" (instead of skipping).
76
+ # Default false preserves historical behavior — meta envelopes only
77
+ # matter to tools that explicitly opt in (typically sinks like
78
+ # write-file-stream when used as a meta-output target).
79
+ accept_meta = bool(
80
+ isinstance(self.settings, dict) and self.settings.get("accept_meta") is True
81
+ )
74
82
 
75
83
  # Main stdin loop
76
84
  for line in sys.stdin:
@@ -81,14 +89,22 @@ class Runtime:
81
89
  if envelope is None:
82
90
  continue
83
91
 
84
- # Skip non-data lines
85
- if envelope.get("t") != "d":
92
+ # Default: only data envelopes. With accept_meta=True, also
93
+ # accept t:"m". Anything else (unknown / missing t) skipped.
94
+ t = envelope.get("t")
95
+ if t != "d" and not (accept_meta and t == "m"):
86
96
  continue
87
97
 
88
98
  id = envelope.get("id", "")
89
99
  src = envelope.get("src", "")
90
100
  v = envelope.get("v", {})
91
101
 
102
+ # Emit an `input` event BEFORE the processor runs so the runner
103
+ # can count rows_in for every stage that reads stdin — including
104
+ # pass-through tools and terminal sinks that never call
105
+ # ctx.output() (and therefore wouldn't otherwise emit a trace).
106
+ write_input(id, src)
107
+
92
108
  ctx = self.create_ctx(id, src)
93
109
  try:
94
110
  self.input_fn(v, self.settings, ctx)
@@ -8,7 +8,7 @@ version = "0.1.0"
8
8
  description = "{{description}}"
9
9
  requires-python = ">=3.11"
10
10
  dependencies = [
11
- "combycode-dpe @ file:///{{framework_abs_path}}",
11
+ "combycode-dpe>=2,<3",
12
12
  ]
13
13
 
14
14
  [project.optional-dependencies]
@@ -0,0 +1,175 @@
1
+ """Tests for the cache helper. Mirror coverage of the TS suite."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import tempfile
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+
11
+ from dpe._cache import cache_path, cached_impl, read_cache_mode
12
+
13
+
14
+ @pytest.fixture
15
+ def storage_dir(monkeypatch):
16
+ """Set DPE_STORAGE to a tempdir for the test, clear DPE_CACHE_MODE."""
17
+ with tempfile.TemporaryDirectory(prefix="dpe-cache-test-") as d:
18
+ monkeypatch.setenv("DPE_STORAGE", d)
19
+ monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
20
+ yield d
21
+
22
+
23
+ # ─── read_cache_mode ─────────────────────────────────────────────────────
24
+
25
+
26
+ def test_read_cache_mode_default(monkeypatch):
27
+ monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
28
+ assert read_cache_mode() == "use"
29
+
30
+
31
+ @pytest.mark.parametrize("m", ["use", "refresh", "bypass", "off"])
32
+ def test_read_cache_mode_recognized(monkeypatch, m):
33
+ monkeypatch.setenv("DPE_CACHE_MODE", m)
34
+ assert read_cache_mode() == m
35
+
36
+
37
+ def test_read_cache_mode_garbage(monkeypatch):
38
+ monkeypatch.setenv("DPE_CACHE_MODE", "nonsense")
39
+ assert read_cache_mode() == "use"
40
+
41
+
42
+ # ─── cache_path ──────────────────────────────────────────────────────────
43
+
44
+
45
+ def test_cache_path_none_when_no_storage(monkeypatch):
46
+ monkeypatch.delenv("DPE_STORAGE", raising=False)
47
+ assert cache_path("ns", {"k": 1}) is None
48
+
49
+
50
+ def test_cache_path_key_order_stable(storage_dir):
51
+ a = cache_path("ns", {"k": 1, "m": 2})
52
+ b = cache_path("ns", {"m": 2, "k": 1})
53
+ assert a == b
54
+
55
+
56
+ def test_cache_path_different_keys(storage_dir):
57
+ assert cache_path("ns", {"k": 1}) != cache_path("ns", {"k": 2})
58
+
59
+
60
+ def test_cache_path_different_namespaces(storage_dir):
61
+ assert cache_path("ns1", {"k": 1}) != cache_path("ns2", {"k": 1})
62
+
63
+
64
+ # ─── cached_impl ─────────────────────────────────────────────────────────
65
+
66
+
67
+ def test_miss_calls_produce_writes_file(storage_dir):
68
+ calls = [0]
69
+
70
+ def produce():
71
+ calls[0] += 1
72
+ return {"hello": "world"}
73
+
74
+ result = cached_impl("ns", {"k": 1}, produce)
75
+ assert result == {"hello": "world"}
76
+ assert calls[0] == 1
77
+ assert Path(cache_path("ns", {"k": 1})).is_file()
78
+
79
+
80
+ def test_hit_skips_produce(storage_dir):
81
+ # Prime via miss to create the dir.
82
+ cached_impl("ns", {"k": 1}, lambda: {"initial": True})
83
+ # Overwrite with our seeded value.
84
+ Path(cache_path("ns", {"k": 1})).write_text(json.dumps({"cached": True}))
85
+
86
+ calls = [0]
87
+
88
+ def produce():
89
+ calls[0] += 1
90
+ return {"fresh": True}
91
+
92
+ result = cached_impl("ns", {"k": 1}, produce)
93
+ assert result == {"cached": True}
94
+ assert calls[0] == 0
95
+
96
+
97
+ def test_refresh_always_produces_overwrites(storage_dir, monkeypatch):
98
+ cached_impl("ns", {"k": 1}, lambda: {"first": True})
99
+ monkeypatch.setenv("DPE_CACHE_MODE", "refresh")
100
+
101
+ calls = [0]
102
+
103
+ def produce():
104
+ calls[0] += 1
105
+ return {"fresh": True}
106
+
107
+ result = cached_impl("ns", {"k": 1}, produce)
108
+ assert result == {"fresh": True}
109
+ assert calls[0] == 1
110
+ on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
111
+ assert on_disk == {"fresh": True}
112
+
113
+
114
+ def test_bypass_no_read_no_write(storage_dir, monkeypatch):
115
+ cached_impl("ns", {"k": 1}, lambda: {"initial": True})
116
+ monkeypatch.setenv("DPE_CACHE_MODE", "bypass")
117
+
118
+ result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
119
+ assert result == {"fresh": True}
120
+ on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
121
+ # bypass does NOT write — original cache content unchanged.
122
+ assert on_disk == {"initial": True}
123
+
124
+
125
+ def test_off_same_as_bypass(storage_dir, monkeypatch):
126
+ cached_impl("ns", {"k": 1}, lambda: {"initial": True})
127
+ monkeypatch.setenv("DPE_CACHE_MODE", "off")
128
+ result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
129
+ assert result == {"fresh": True}
130
+ on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
131
+ assert on_disk == {"initial": True}
132
+
133
+
134
+ def test_no_storage_disables_cache(monkeypatch):
135
+ monkeypatch.delenv("DPE_STORAGE", raising=False)
136
+ monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
137
+
138
+ calls = [0]
139
+
140
+ def produce():
141
+ calls[0] += 1
142
+ return {"n": calls[0]}
143
+
144
+ r1 = cached_impl("ns", {"k": 1}, produce)
145
+ r2 = cached_impl("ns", {"k": 1}, produce)
146
+ assert r1 == {"n": 1}
147
+ assert r2 == {"n": 2}
148
+
149
+
150
+ def test_producer_error_propagates_no_write(storage_dir):
151
+ attempts = [0]
152
+
153
+ def boom():
154
+ attempts[0] += 1
155
+ raise RuntimeError("kaboom")
156
+
157
+ with pytest.raises(RuntimeError, match="kaboom"):
158
+ cached_impl("ns", {"k": 1}, boom)
159
+ assert attempts[0] == 1
160
+ assert not Path(cache_path("ns", {"k": 1})).is_file()
161
+
162
+
163
+ def test_malformed_cache_treated_as_miss(storage_dir):
164
+ cached_impl("ns", {"k": 1}, lambda: {"ok": True})
165
+ Path(cache_path("ns", {"k": 1})).write_text("this is not json")
166
+
167
+ calls = [0]
168
+
169
+ def produce():
170
+ calls[0] += 1
171
+ return {"recovered": True}
172
+
173
+ result = cached_impl("ns", {"k": 1}, produce)
174
+ assert result == {"recovered": True}
175
+ assert calls[0] == 1
@@ -20,7 +20,9 @@ import pytest
20
20
 
21
21
  from dpe import _envelope as env
22
22
 
23
- SCHEMA_PATH = Path(__file__).resolve().parents[3] / "runner" / "schemas" / "stderr-events.schema.json"
23
+ SCHEMA_PATH = (
24
+ Path(__file__).resolve().parents[3] / "runner" / "schemas" / "stderr-events.schema.json"
25
+ )
24
26
 
25
27
 
26
28
  @pytest.fixture(scope="module")