combycode-dpe 2.0.0rc3__tar.gz → 2.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/PKG-INFO +1 -1
  2. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/README.md +3 -1
  3. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/pyproject.toml +1 -1
  4. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/__init__.py +2 -0
  5. combycode_dpe-2.0.2/src/dpe/_cache.py +81 -0
  6. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_context.py +61 -9
  7. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_envelope.py +27 -4
  8. combycode_dpe-2.0.2/src/dpe/_paths.py +108 -0
  9. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_runtime.py +25 -5
  10. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/pyproject.toml +1 -1
  11. combycode_dpe-2.0.2/tests/test_cache.py +175 -0
  12. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/tests/test_integration.py +97 -0
  13. combycode_dpe-2.0.2/tests/test_paths.py +131 -0
  14. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/tests/test_stderr_events_conformance.py +3 -1
  15. combycode_dpe-2.0.0rc3/template/verify/case-basic/expected.ndjson +0 -2
  16. combycode_dpe-2.0.0rc3/template/verify/case-basic/input.ndjson +0 -2
  17. combycode_dpe-2.0.0rc3/template/verify/case-basic/settings.json +0 -1
  18. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/.gitignore +0 -0
  19. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_accumulators.py +0 -0
  20. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/meta.json +0 -0
  21. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/spec.yaml +0 -0
  22. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/src/__PKG__/__init__.py +0 -0
  23. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/src/__PKG__/main.py +0 -0
  24. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/tests/test_basic.py +0 -0
  25. {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/tests/test_accumulators.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: combycode-dpe
3
- Version: 2.0.0rc3
3
+ Version: 2.0.2
4
4
  Summary: DPE framework for building streaming pipeline tools in Python
5
5
  Project-URL: Repository, https://github.com/combycode/dpe-framework-python
6
6
  License-Expression: AGPL-3.0-or-later
@@ -91,9 +91,11 @@ dpe-dev scaffold --name my-tool --runtime python --out ./my-tool --description "
91
91
  cd my-tool
92
92
  dpe-dev build . # creates .venv, installs the tool with [dev] extras
93
93
  dpe-dev test . # pytest inside the venv
94
- dpe-dev verify . # spawn the tool's main.py, feed input.ndjson, diff stdout vs expected.ndjson
95
94
  ```
96
95
 
96
+ End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
97
+ tool is wired into a pipeline — see `docs/testing.md`.
98
+
97
99
  ## Optional speed-up
98
100
 
99
101
  For bulk-hashing payloads, install `xxhash` and pass `algorithm="xxhash"`
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "combycode-dpe"
7
- version = "2.0.0rc3"
7
+ version = "2.0.2"
8
8
  description = "DPE framework for building streaming pipeline tools in Python"
9
9
  license = "AGPL-3.0-or-later"
10
10
  requires-python = ">=3.11"
@@ -36,6 +36,7 @@ from dpe._accumulators import (
36
36
  )
37
37
  from dpe._context import Context
38
38
  from dpe._envelope import hash_file, hash_string
39
+ from dpe._paths import EnvPaths
39
40
  from dpe._runtime import Runtime
40
41
 
41
42
 
@@ -63,6 +64,7 @@ __all__ = [
63
64
  "hash_string",
64
65
  "hash_file",
65
66
  "Context",
67
+ "EnvPaths",
66
68
  "Memory",
67
69
  "Number",
68
70
  "Average",
@@ -0,0 +1,81 @@
1
+ """Cache helper used by Context.cached(). Mirrors the TS framework's
2
+ cache.ts behavior — same DPE_CACHE_MODE semantics, same on-disk layout
3
+ ($DPE_STORAGE/<namespace>/<hash>.json), same canonical-JSON key
4
+ hashing.
5
+
6
+ Failure modes (cache-disabling, NOT errors propagated to user):
7
+ - DPE_STORAGE not set -> cache disabled, every call produces
8
+ - cache file unreadable -> treat as miss, log warn
9
+ - cache file unparseable -> treat as miss, log warn
10
+ - producer raises -> re-raise to caller (no cache write)
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import hashlib
16
+ import json
17
+ import os
18
+ from collections.abc import Callable
19
+ from typing import Any, TypeVar
20
+
21
+ from dpe._envelope import write_log
22
+
23
+ T = TypeVar("T")
24
+
25
+ CACHE_MODES = ("use", "refresh", "bypass", "off")
26
+
27
+
28
+ def read_cache_mode() -> str:
29
+ """Read DPE_CACHE_MODE; default to "use". Unrecognized -> "use"."""
30
+ v = os.getenv("DPE_CACHE_MODE", "use")
31
+ return v if v in CACHE_MODES else "use"
32
+
33
+
34
+ def cache_path(namespace: str, key: Any) -> str | None:
35
+ """Compute the on-disk path for (namespace, key). Returns None when
36
+ DPE_STORAGE isn't set -- caller treats that as cache disabled.
37
+ """
38
+ storage = os.getenv("DPE_STORAGE")
39
+ if not storage:
40
+ return None
41
+ canonical = json.dumps(key, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
42
+ h = hashlib.blake2b(canonical.encode("utf-8")).hexdigest()[:32]
43
+ return os.path.join(storage, namespace, f"{h}.json")
44
+
45
+
46
+ def cached_impl(
47
+ namespace: str,
48
+ key: Any,
49
+ produce: Callable[[], T],
50
+ ) -> T:
51
+ """Read-or-produce cache around `produce`. Honors the four cache
52
+ modes. Returns the produced or cached value."""
53
+ mode = read_cache_mode()
54
+ path = cache_path(namespace, key)
55
+
56
+ can_read = path is not None and mode in ("use", "refresh")
57
+ can_write = path is not None and mode not in ("bypass", "off")
58
+ will_read = can_read and mode != "refresh"
59
+
60
+ if will_read and path is not None and os.path.isfile(path):
61
+ try:
62
+ with open(path, encoding="utf-8") as f:
63
+ value = json.load(f)
64
+ write_log(f"cached: hit ({namespace})", "debug")
65
+ return value
66
+ except (OSError, json.JSONDecodeError) as e:
67
+ write_log(f"cached: read failed ({namespace}) — {e}", "warn")
68
+ # Fall through to produce.
69
+
70
+ result = produce()
71
+
72
+ if can_write and path is not None:
73
+ try:
74
+ os.makedirs(os.path.dirname(path), exist_ok=True)
75
+ with open(path, "w", encoding="utf-8") as f:
76
+ json.dump(result, f, ensure_ascii=False, separators=(",", ":"))
77
+ except OSError as e:
78
+ write_log(f"cached: write failed ({namespace}) — {e}", "warn")
79
+ # Don't fail the caller — they got their value.
80
+
81
+ return result
@@ -1,6 +1,10 @@
1
1
  """Context object — passed to every processor invocation."""
2
2
 
3
+ from collections.abc import Callable
4
+ from typing import Any, TypeVar
5
+
3
6
  from dpe._accumulators import Memory
7
+ from dpe._cache import cached_impl
4
8
  from dpe._envelope import (
5
9
  hash_file,
6
10
  hash_string,
@@ -11,6 +15,9 @@ from dpe._envelope import (
11
15
  write_stats,
12
16
  write_trace,
13
17
  )
18
+ from dpe._paths import EnvPaths
19
+
20
+ T = TypeVar("T")
14
21
 
15
22
 
16
23
  class Context:
@@ -21,13 +28,15 @@ class Context:
21
28
  Memory is shared across all processors in the tool.
22
29
  """
23
30
 
24
- __slots__ = ("id", "src", "memory", "_runtime", "_labels")
31
+ __slots__ = ("id", "src", "memory", "_runtime", "_paths", "_labels")
25
32
 
26
- def __init__(self, id: str, src: str, memory: Memory, runtime):
33
+ def __init__(self, id: str, src: str, memory: Memory, runtime,
34
+ paths: EnvPaths | None = None):
27
35
  self.id = id
28
36
  self.src = src
29
37
  self.memory = memory
30
38
  self._runtime = runtime
39
+ self._paths = paths if paths is not None else EnvPaths()
31
40
  # Labels accumulated by ctx.trace(k, v). Flushed as one merged
32
41
  # {type:"trace"} stderr event before each ctx.output(), then cleared.
33
42
  self._labels: dict = {}
@@ -50,15 +59,17 @@ class Context:
50
59
  def output(self, v, *, id: str | None = None, src: str | None = None):
51
60
  """Emit data record to stdout.
52
61
 
53
- Flushes accumulated trace labels as a merged trace event first,
54
- then writes the envelope, then clears the label bag.
62
+ Flushes accumulated trace labels as a merged trace event first
63
+ (channel="data" -> counts as rows_out at the runner), then writes
64
+ the envelope, then clears the label bag.
65
+ Absolute paths in v are reverse-tokenized to $token/... form.
55
66
  """
56
67
  out_id = id if id is not None else self.id
57
68
  out_src = src if src is not None else self.src
58
- # Emit trace even with empty labels — the chain row itself is the value.
59
- write_trace(out_id, out_src, self._labels)
69
+ # Emit trace even with empty labels -- the chain row itself is the value.
70
+ write_trace(out_id, out_src, self._labels, channel="data")
60
71
  self._labels = {}
61
- write_data(v, out_id, out_src)
72
+ write_data(self._paths.tokenize_value(v), out_id, out_src)
62
73
 
63
74
  def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
64
75
  """Emit to internal named queue. Processed by process_<queue>().
@@ -82,8 +93,14 @@ class Context:
82
93
  self._runtime.drain_queue()
83
94
 
84
95
  def meta(self, v: dict):
85
- """Emit metadata record to stdout."""
86
- write_meta(v)
96
+ """Emit metadata record to stdout.
97
+
98
+ Also emits a {type:"trace", channel:"meta"} stderr event so the
99
+ runner can increment its per-stage `meta` counter. Inherits
100
+ ctx id/src. Absolute paths in v are reverse-tokenized to $token/... form.
101
+ """
102
+ write_trace(self.id, self.src, {}, channel="meta")
103
+ write_meta(self._paths.tokenize_value(v))
87
104
 
88
105
  def log(self, msg: str, *, level: str = "info", **extra):
89
106
  """Write structured log to stderr."""
@@ -100,3 +117,38 @@ class Context:
100
117
  def hash_file(self, filepath: str, algorithm: str = "blake2b") -> str | None:
101
118
  """Hash file content in chunks. Returns hex string or None on error."""
102
119
  return hash_file(filepath, algorithm)
120
+
121
+ def cached(
122
+ self,
123
+ namespace: str,
124
+ key: Any,
125
+ produce: Callable[[], T],
126
+ ) -> T:
127
+ """Cache the result of `produce()` under
128
+ `$DPE_STORAGE/<namespace>/<hash>.json`.
129
+
130
+ Honors the runner's `DPE_CACHE_MODE` env:
131
+ - use (default) -- read cache if present, else produce + write
132
+ - refresh -- always produce, overwrite cache
133
+ - bypass -- produce, skip both read and write
134
+ - off -- same as bypass
135
+
136
+ `key` is canonical-JSON-hashed (blake2b, first 32 hex chars).
137
+ Compose it from whatever determines output equivalence.
138
+
139
+ If `$DPE_STORAGE` isn't set (e.g. tool invoked outside a
140
+ pipeline), cache is silently disabled — every call produces.
141
+
142
+ Producer errors propagate; failed runs do NOT poison the cache.
143
+
144
+ Example:
145
+ result = ctx.cached(
146
+ "doc-converter",
147
+ {"file_hash": ctx.hash_file(v["path"]),
148
+ "settings": settings,
149
+ "page": page_idx},
150
+ lambda: provider.convert_page(...),
151
+ )
152
+ ctx.output(result)
153
+ """
154
+ return cached_impl(namespace, key, produce)
@@ -26,7 +26,12 @@ def hash_file(filepath: str, algorithm: str = "blake2b", chunk_size: int = 65536
26
26
  try:
27
27
  if algorithm == "xxhash":
28
28
  try:
29
- import xxhash # type: ignore[import-not-found]
29
+ # xxhash is an optional runtime dep (see fallback). Need
30
+ # both codes here: `import-not-found` fires when xxhash isn't
31
+ # installed (CI), `unused-ignore` fires when it IS installed
32
+ # (some dev machines). Listing both keeps mypy happy in both
33
+ # environments without disabling the warning globally.
34
+ import xxhash # type: ignore[import-not-found,unused-ignore]
30
35
  h = xxhash.xxh3_64()
31
36
  except ImportError:
32
37
  h = hashlib.blake2b(digest_size=16)
@@ -89,9 +94,27 @@ def write_error(v, err, id: str, src: str):
89
94
  _write_stderr(record)
90
95
 
91
96
 
92
- def write_trace(id: str, src: str, labels: dict):
93
- """Write merged trace event to stderr. Called by ctx before each output."""
94
- _write_stderr({"type": "trace", "id": id, "src": src, "labels": labels})
97
+ def write_trace(id: str, src: str, labels: dict, channel: str | None = None):
98
+ """Write merged trace event to stderr. Called by ctx before each output.
99
+
100
+ `channel` distinguishes counter buckets at the runner side:
101
+ - "data" from ctx.output -> rows_out
102
+ - "meta" from ctx.meta -> meta
103
+ None is written without a `channel` key (back-compat: runner treats
104
+ unset as data).
105
+ """
106
+ record: dict = {"type": "trace", "id": id, "src": src, "labels": labels}
107
+ if channel is not None:
108
+ record["channel"] = channel
109
+ _write_stderr(record)
110
+
111
+
112
+ def write_input(id: str, src: str):
113
+ """Write input event to stderr. Emitted by the runtime per envelope read
114
+ from stdin, BEFORE the processor runs. Lets the runner count rows_in
115
+ for stages that never call ctx.output() (pass-through, terminal sinks).
116
+ """
117
+ _write_stderr({"type": "input", "id": id, "src": src})
95
118
 
96
119
 
97
120
  def write_stats(data: dict):
@@ -0,0 +1,108 @@
1
+ """EnvPaths — resolves $token path prefixes in envelope v on input;
2
+ reverse-tokenizes absolute paths back to $token form on output.
3
+
4
+ Input side (runtime, before processor): $token/subpath -> absolute path
5
+ Output side (context.output / context.meta): absolute path -> $token/subpath
6
+
7
+ Both sides are a no-op when the DPE_* env vars are not set.
8
+ """
9
+
10
+ import os
11
+ from typing import Any
12
+
13
+ _TOKEN_MAP = [
14
+ ("input", "DPE_INPUT"),
15
+ ("output", "DPE_OUTPUT"),
16
+ ("configs", "DPE_CONFIGS"),
17
+ ("storage", "DPE_STORAGE"),
18
+ ("temp", "DPE_TEMP"),
19
+ ("session", "DPE_SESSION"),
20
+ ]
21
+
22
+
23
+ class EnvPaths:
24
+ """Resolved env prefix map. Build once at tool startup."""
25
+
26
+ __slots__ = ("_entries",)
27
+
28
+ def __init__(self, pairs: list[tuple[str, str]] | None = None):
29
+ """
30
+ If *pairs* is None the standard DPE_* env vars are read.
31
+ Pass explicit pairs for tests: ``[("input", "/abs/input"), ...]``.
32
+
33
+ Prefer the named factories :meth:`from_env` and :meth:`from_pairs`
34
+ -- they're consistent with the Rust SDK (``from_env()`` /
35
+ ``from_pairs(&[...])``) and read more clearly at call sites. The
36
+ polymorphic constructor is kept for backwards compat.
37
+ """
38
+ if pairs is None:
39
+ raw = []
40
+ for token, env_var in _TOKEN_MAP:
41
+ val = os.environ.get(env_var, "")
42
+ if val:
43
+ raw.append((token, val.replace("\\", "/")))
44
+ else:
45
+ raw = [(t, a.replace("\\", "/")) for t, a in pairs]
46
+ # Sort longest-abs-first for greedy tokenization.
47
+ self._entries: list[tuple[str, str]] = sorted(
48
+ raw, key=lambda e: len(e[1]), reverse=True
49
+ )
50
+
51
+ @classmethod
52
+ def from_env(cls) -> "EnvPaths":
53
+ """Read the standard DPE_* env vars; missing/empty vars are skipped."""
54
+ return cls()
55
+
56
+ @classmethod
57
+ def from_pairs(cls, pairs: list[tuple[str, str]]) -> "EnvPaths":
58
+ """Construct from explicit (token, abs_path) pairs -- useful for tests."""
59
+ return cls(pairs)
60
+
61
+ def is_empty(self) -> bool:
62
+ return not self._entries
63
+
64
+ def resolve_value(self, v: Any) -> Any:
65
+ """Walk v, resolving $token[/subpath] strings to absolute paths."""
66
+ if self.is_empty():
67
+ return v
68
+ return self._walk(v, self._resolve_str)
69
+
70
+ def tokenize_value(self, v: Any) -> Any:
71
+ """Walk v, tokenizing absolute paths back to $token[/subpath] form."""
72
+ if self.is_empty():
73
+ return v
74
+ return self._walk(v, self._tokenize_str)
75
+
76
+ def _walk(self, v: Any, f) -> Any:
77
+ if isinstance(v, str):
78
+ return f(v)
79
+ if isinstance(v, list):
80
+ return [self._walk(x, f) for x in v]
81
+ if isinstance(v, dict):
82
+ return {k: self._walk(val, f) for k, val in v.items()}
83
+ return v
84
+
85
+ def _resolve_str(self, s: str) -> str:
86
+ if not s.startswith("$"):
87
+ return s
88
+ slash = s.find("/")
89
+ if slash < 0:
90
+ name, tail = s[1:], ""
91
+ else:
92
+ name, tail = s[1:slash], s[slash:]
93
+ for token, abs_path in self._entries:
94
+ if token == name:
95
+ return abs_path + tail if tail else abs_path
96
+ return s
97
+
98
+ def _tokenize_str(self, s: str) -> str:
99
+ normalized = s.replace("\\", "/")
100
+ for token, abs_path in self._entries:
101
+ if normalized.startswith(abs_path):
102
+ rest = normalized[len(abs_path):]
103
+ if not rest:
104
+ return f"${token}"
105
+ if rest.startswith("/"):
106
+ return f"${token}{rest}"
107
+ # rest doesn't start with "/" -- partial component match, skip
108
+ return s
@@ -9,7 +9,8 @@ from collections.abc import Callable
9
9
 
10
10
  from dpe._accumulators import Memory
11
11
  from dpe._context import Context
12
- from dpe._envelope import parse_envelope, write_log
12
+ from dpe._envelope import parse_envelope, write_input, write_log
13
+ from dpe._paths import EnvPaths
13
14
 
14
15
 
15
16
  class Runtime:
@@ -22,6 +23,7 @@ class Runtime:
22
23
  self.processors: dict[str, Callable] = {}
23
24
  self._queue: deque = deque()
24
25
  self._shutdown = False
26
+ self._paths = EnvPaths()
25
27
 
26
28
  def enqueue(self, queue_name: str, v, id: str, src: str):
27
29
  """Add item to internal queue."""
@@ -40,7 +42,7 @@ class Runtime:
40
42
  write_log(f"No processor for queue '{name}', dropping item", "warn")
41
43
  continue
42
44
 
43
- ctx = Context(item_id, item_src, self.memory, self)
45
+ ctx = Context(item_id, item_src, self.memory, self, self._paths)
44
46
  try:
45
47
  proc(v, self.settings, ctx)
46
48
  except Exception as e:
@@ -52,7 +54,7 @@ class Runtime:
52
54
  write_log(f"Queue drain hit safety limit ({max_iterations})", "error")
53
55
 
54
56
  def create_ctx(self, id: str, src: str) -> Context:
55
- return Context(id, src, self.memory, self)
57
+ return Context(id, src, self.memory, self, self._paths)
56
58
 
57
59
  def _handle_signal(self, signum, frame):
58
60
  self._shutdown = True
@@ -71,6 +73,14 @@ class Runtime:
71
73
 
72
74
  # Parse settings
73
75
  self.settings = _parse_settings()
76
+ # `accept_meta: true` in settings → the read loop dispatches t:"m"
77
+ # envelopes to process_input alongside t:"d" (instead of skipping).
78
+ # Default false preserves historical behavior — meta envelopes only
79
+ # matter to tools that explicitly opt in (typically sinks like
80
+ # write-file-stream when used as a meta-output target).
81
+ accept_meta = bool(
82
+ isinstance(self.settings, dict) and self.settings.get("accept_meta") is True
83
+ )
74
84
 
75
85
  # Main stdin loop
76
86
  for line in sys.stdin:
@@ -81,13 +91,23 @@ class Runtime:
81
91
  if envelope is None:
82
92
  continue
83
93
 
84
- # Skip non-data lines
85
- if envelope.get("t") != "d":
94
+ # Default: only data envelopes. With accept_meta=True, also
95
+ # accept t:"m". Anything else (unknown / missing t) skipped.
96
+ t = envelope.get("t")
97
+ if t != "d" and not (accept_meta and t == "m"):
86
98
  continue
87
99
 
88
100
  id = envelope.get("id", "")
89
101
  src = envelope.get("src", "")
90
102
  v = envelope.get("v", {})
103
+ # Resolve $token paths in v before handing to processor.
104
+ v = self._paths.resolve_value(v)
105
+
106
+ # Emit an `input` event BEFORE the processor runs so the runner
107
+ # can count rows_in for every stage that reads stdin — including
108
+ # pass-through tools and terminal sinks that never call
109
+ # ctx.output() (and therefore wouldn't otherwise emit a trace).
110
+ write_input(id, src)
91
111
 
92
112
  ctx = self.create_ctx(id, src)
93
113
  try:
@@ -8,7 +8,7 @@ version = "0.1.0"
8
8
  description = "{{description}}"
9
9
  requires-python = ">=3.11"
10
10
  dependencies = [
11
- "combycode-dpe @ file:///{{framework_abs_path}}",
11
+ "combycode-dpe>=2,<3",
12
12
  ]
13
13
 
14
14
  [project.optional-dependencies]
@@ -0,0 +1,175 @@
1
+ """Tests for the cache helper. Mirror coverage of the TS suite."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import tempfile
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+
11
+ from dpe._cache import cache_path, cached_impl, read_cache_mode
12
+
13
+
14
+ @pytest.fixture
15
+ def storage_dir(monkeypatch):
16
+ """Set DPE_STORAGE to a tempdir for the test, clear DPE_CACHE_MODE."""
17
+ with tempfile.TemporaryDirectory(prefix="dpe-cache-test-") as d:
18
+ monkeypatch.setenv("DPE_STORAGE", d)
19
+ monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
20
+ yield d
21
+
22
+
23
+ # ─── read_cache_mode ─────────────────────────────────────────────────────
24
+
25
+
26
+ def test_read_cache_mode_default(monkeypatch):
27
+ monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
28
+ assert read_cache_mode() == "use"
29
+
30
+
31
+ @pytest.mark.parametrize("m", ["use", "refresh", "bypass", "off"])
32
+ def test_read_cache_mode_recognized(monkeypatch, m):
33
+ monkeypatch.setenv("DPE_CACHE_MODE", m)
34
+ assert read_cache_mode() == m
35
+
36
+
37
+ def test_read_cache_mode_garbage(monkeypatch):
38
+ monkeypatch.setenv("DPE_CACHE_MODE", "nonsense")
39
+ assert read_cache_mode() == "use"
40
+
41
+
42
+ # ─── cache_path ──────────────────────────────────────────────────────────
43
+
44
+
45
+ def test_cache_path_none_when_no_storage(monkeypatch):
46
+ monkeypatch.delenv("DPE_STORAGE", raising=False)
47
+ assert cache_path("ns", {"k": 1}) is None
48
+
49
+
50
+ def test_cache_path_key_order_stable(storage_dir):
51
+ a = cache_path("ns", {"k": 1, "m": 2})
52
+ b = cache_path("ns", {"m": 2, "k": 1})
53
+ assert a == b
54
+
55
+
56
+ def test_cache_path_different_keys(storage_dir):
57
+ assert cache_path("ns", {"k": 1}) != cache_path("ns", {"k": 2})
58
+
59
+
60
+ def test_cache_path_different_namespaces(storage_dir):
61
+ assert cache_path("ns1", {"k": 1}) != cache_path("ns2", {"k": 1})
62
+
63
+
64
+ # ─── cached_impl ─────────────────────────────────────────────────────────
65
+
66
+
67
+ def test_miss_calls_produce_writes_file(storage_dir):
68
+ calls = [0]
69
+
70
+ def produce():
71
+ calls[0] += 1
72
+ return {"hello": "world"}
73
+
74
+ result = cached_impl("ns", {"k": 1}, produce)
75
+ assert result == {"hello": "world"}
76
+ assert calls[0] == 1
77
+ assert Path(cache_path("ns", {"k": 1})).is_file()
78
+
79
+
80
+ def test_hit_skips_produce(storage_dir):
81
+ # Prime via miss to create the dir.
82
+ cached_impl("ns", {"k": 1}, lambda: {"initial": True})
83
+ # Overwrite with our seeded value.
84
+ Path(cache_path("ns", {"k": 1})).write_text(json.dumps({"cached": True}))
85
+
86
+ calls = [0]
87
+
88
+ def produce():
89
+ calls[0] += 1
90
+ return {"fresh": True}
91
+
92
+ result = cached_impl("ns", {"k": 1}, produce)
93
+ assert result == {"cached": True}
94
+ assert calls[0] == 0
95
+
96
+
97
+ def test_refresh_always_produces_overwrites(storage_dir, monkeypatch):
98
+ cached_impl("ns", {"k": 1}, lambda: {"first": True})
99
+ monkeypatch.setenv("DPE_CACHE_MODE", "refresh")
100
+
101
+ calls = [0]
102
+
103
+ def produce():
104
+ calls[0] += 1
105
+ return {"fresh": True}
106
+
107
+ result = cached_impl("ns", {"k": 1}, produce)
108
+ assert result == {"fresh": True}
109
+ assert calls[0] == 1
110
+ on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
111
+ assert on_disk == {"fresh": True}
112
+
113
+
114
+ def test_bypass_no_read_no_write(storage_dir, monkeypatch):
115
+ cached_impl("ns", {"k": 1}, lambda: {"initial": True})
116
+ monkeypatch.setenv("DPE_CACHE_MODE", "bypass")
117
+
118
+ result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
119
+ assert result == {"fresh": True}
120
+ on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
121
+ # bypass does NOT write — original cache content unchanged.
122
+ assert on_disk == {"initial": True}
123
+
124
+
125
+ def test_off_same_as_bypass(storage_dir, monkeypatch):
126
+ cached_impl("ns", {"k": 1}, lambda: {"initial": True})
127
+ monkeypatch.setenv("DPE_CACHE_MODE", "off")
128
+ result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
129
+ assert result == {"fresh": True}
130
+ on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
131
+ assert on_disk == {"initial": True}
132
+
133
+
134
+ def test_no_storage_disables_cache(monkeypatch):
135
+ monkeypatch.delenv("DPE_STORAGE", raising=False)
136
+ monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
137
+
138
+ calls = [0]
139
+
140
+ def produce():
141
+ calls[0] += 1
142
+ return {"n": calls[0]}
143
+
144
+ r1 = cached_impl("ns", {"k": 1}, produce)
145
+ r2 = cached_impl("ns", {"k": 1}, produce)
146
+ assert r1 == {"n": 1}
147
+ assert r2 == {"n": 2}
148
+
149
+
150
+ def test_producer_error_propagates_no_write(storage_dir):
151
+ attempts = [0]
152
+
153
+ def boom():
154
+ attempts[0] += 1
155
+ raise RuntimeError("kaboom")
156
+
157
+ with pytest.raises(RuntimeError, match="kaboom"):
158
+ cached_impl("ns", {"k": 1}, boom)
159
+ assert attempts[0] == 1
160
+ assert not Path(cache_path("ns", {"k": 1})).is_file()
161
+
162
+
163
+ def test_malformed_cache_treated_as_miss(storage_dir):
164
+ cached_impl("ns", {"k": 1}, lambda: {"ok": True})
165
+ Path(cache_path("ns", {"k": 1})).write_text("this is not json")
166
+
167
+ calls = [0]
168
+
169
+ def produce():
170
+ calls[0] += 1
171
+ return {"recovered": True}
172
+
173
+ result = cached_impl("ns", {"k": 1}, produce)
174
+ assert result == {"recovered": True}
175
+ assert calls[0] == 1
@@ -1,6 +1,7 @@
1
1
  """Integration tests — full tool lifecycle via subprocess."""
2
2
 
3
3
  import json
4
+ import os
4
5
  import subprocess
5
6
  import sys
6
7
  import tempfile
@@ -546,3 +547,99 @@ class TestContext:
546
547
 
547
548
  assert data[0]["v"]["id"] == "my-id"
548
549
  assert data[0]["v"]["src"] == "my-src"
550
+
551
+
552
+ class TestEnvPathsRoundTrip:
553
+ def _run_with_env(self, code, settings, stdin_lines, env_overrides):
554
+ """Like run_tool_code but injects custom DPE_* env vars into the subprocess."""
555
+ import subprocess
556
+ import sys
557
+ import tempfile
558
+ import textwrap
559
+ from pathlib import Path
560
+
561
+ stdin_data = (
562
+ "\n".join(json.dumps(r) for r in stdin_lines) + "\n" if stdin_lines else ""
563
+ )
564
+ with tempfile.NamedTemporaryFile(
565
+ mode="w", suffix=".py", delete=False,
566
+ dir=tempfile.gettempdir(), encoding="utf-8",
567
+ ) as f:
568
+ f.write(textwrap.dedent(code))
569
+ f.flush()
570
+ tool_path = f.name
571
+
572
+ env = {**os.environ, **env_overrides}
573
+ result = subprocess.run(
574
+ [sys.executable, tool_path, json.dumps(settings)],
575
+ input=stdin_data, capture_output=True, text=True,
576
+ timeout=30, env=env,
577
+ )
578
+ Path(tool_path).unlink(missing_ok=True)
579
+
580
+ data = []
581
+ for line in result.stdout.strip().split("\n"):
582
+ if line:
583
+ rec = json.loads(line)
584
+ if rec.get("t") == "d":
585
+ data.append(rec)
586
+ return data
587
+
588
+ def test_token_resolved_on_input(self):
589
+ """$input in envelope v is expanded to abs path before processor sees it.
590
+
591
+ The processor inspects the path (not outputs it) so the tokenize-on-output
592
+ step does not reverse the expansion — we test the expansion independently.
593
+ """
594
+ data = self._run_with_env("""
595
+ import dpe
596
+
597
+ def process_input(v, settings, ctx):
598
+ # Check the resolved value; output a boolean, not the path itself,
599
+ # so the tokenize-on-output step doesn't convert it back.
600
+ ctx.output({"starts_abs": v["path"].startswith("/mnt/drive")})
601
+
602
+ if __name__ == "__main__":
603
+ dpe.run()
604
+ """, {}, [inp({"path": "$input/data.csv"})],
605
+ {"DPE_INPUT": "/mnt/drive"})
606
+
607
+ assert data[0]["v"]["starts_abs"] is True
608
+
609
+ def test_abs_path_tokenized_on_output(self):
610
+ """Absolute paths in ctx.output(v) are tokenized back to $token form.
611
+
612
+ The processor hardcodes an absolute path string (not from resolved input)
613
+ so we're testing only the output tokenization.
614
+ """
615
+ data = self._run_with_env("""
616
+ import dpe
617
+
618
+ def process_input(v, settings, ctx):
619
+ # Output a hard-coded abs path -- not from resolved input.
620
+ ctx.output({"path": "/mnt/drive/result.csv"})
621
+
622
+ if __name__ == "__main__":
623
+ dpe.run()
624
+ """, {}, [inp({"x": 1})],
625
+ {"DPE_INPUT": "/mnt/drive"})
626
+
627
+ assert data[0]["v"]["path"] == "$input/result.csv"
628
+
629
+ def test_no_env_passthrough(self):
630
+ """Without DPE_* vars set, $token paths pass through unchanged."""
631
+ env_overrides = {k: "" for k in [
632
+ "DPE_INPUT", "DPE_OUTPUT", "DPE_CONFIGS",
633
+ "DPE_STORAGE", "DPE_TEMP", "DPE_SESSION",
634
+ ]}
635
+ data = self._run_with_env("""
636
+ import dpe
637
+
638
+ def process_input(v, settings, ctx):
639
+ ctx.output(v)
640
+
641
+ if __name__ == "__main__":
642
+ dpe.run()
643
+ """, {}, [inp({"path": "$input/foo"})], env_overrides)
644
+
645
+ assert data[0]["v"]["path"] == "$input/foo"
@@ -0,0 +1,131 @@
1
+ """Tests for EnvPaths -- $token resolve/tokenize round-trip."""
2
+
3
+ from dpe._paths import EnvPaths
4
+
5
+
6
+ def paths():
7
+ return EnvPaths([
8
+ ("input", "/abs/input"),
9
+ ("output", "/abs/output"),
10
+ ("storage", "/abs/storage"),
11
+ # Intentionally shorter abs -- tests longest-match logic.
12
+ ("session", "/abs/storage/session"),
13
+ ])
14
+
15
+
16
+ # --- resolve_value ---
17
+
18
+ class TestResolveValue:
19
+ def test_top_level_token(self):
20
+ assert paths().resolve_value("$input") == "/abs/input"
21
+
22
+ def test_token_with_subpath(self):
23
+ assert paths().resolve_value("$input/data/file.csv") == "/abs/input/data/file.csv"
24
+
25
+ def test_unknown_token_unchanged(self):
26
+ assert paths().resolve_value("$set/field") == "$set/field"
27
+ assert paths().resolve_value("$bogus") == "$bogus"
28
+
29
+ def test_recurses_into_dict(self):
30
+ v = {"path": "$input/a.csv", "n": 42, "nested": {"p": "$output/b"}}
31
+ r = paths().resolve_value(v)
32
+ assert r["path"] == "/abs/input/a.csv"
33
+ assert r["n"] == 42
34
+ assert r["nested"]["p"] == "/abs/output/b"
35
+
36
+ def test_recurses_into_list(self):
37
+ v = ["$input/x", "$output/y", "plain"]
38
+ r = paths().resolve_value(v)
39
+ assert r == ["/abs/input/x", "/abs/output/y", "plain"]
40
+
41
+ def test_no_env_is_noop(self):
42
+ empty = EnvPaths([])
43
+ v = {"p": "$input/foo"}
44
+ assert empty.resolve_value(v) == v
45
+
46
+ def test_non_string_types_unchanged(self):
47
+ v = {"n": 42, "b": True, "nil": None, "p": "$input"}
48
+ r = paths().resolve_value(v)
49
+ assert r["n"] == 42
50
+ assert r["b"] is True
51
+ assert r["nil"] is None
52
+ assert r["p"] == "/abs/input"
53
+
54
+
55
+ # --- tokenize_value ---
56
+
57
+ class TestTokenizeValue:
58
+ def test_exact_prefix(self):
59
+ assert paths().tokenize_value("/abs/input") == "$input"
60
+
61
+ def test_prefix_with_subpath(self):
62
+ assert paths().tokenize_value("/abs/output/results/out.csv") == "$output/results/out.csv"
63
+
64
+ def test_partial_component_not_replaced(self):
65
+ # "/abs/inputXYZ" must NOT match "$input"
66
+ assert paths().tokenize_value("/abs/inputXYZ") == "/abs/inputXYZ"
67
+
68
+ def test_longest_prefix_wins(self):
69
+ # "/abs/storage/session/..." should match "$session", not "$storage"
70
+ assert paths().tokenize_value("/abs/storage/session/data.json") == "$session/data.json"
71
+
72
+ def test_non_path_unchanged(self):
73
+ assert paths().tokenize_value("hello world") == "hello world"
74
+ assert paths().tokenize_value(42) == 42
75
+ assert paths().tokenize_value(True) is True
76
+ assert paths().tokenize_value(None) is None
77
+
78
+ def test_recurses_into_nested(self):
79
+ v = {"a": {"b": "/abs/input/x"}, "arr": ["/abs/output/y"]}
80
+ r = paths().tokenize_value(v)
81
+ assert r["a"]["b"] == "$input/x"
82
+ assert r["arr"][0] == "$output/y"
83
+
84
+
85
+ # --- round-trip ---
86
+
87
+ class TestRoundTrip:
88
+ def test_resolve_then_tokenize(self):
89
+ p = EnvPaths([("input", "/data/in"), ("output", "/data/out")])
90
+ original = {"src": "$input/file.csv", "dst": "$output/result.csv"}
91
+ resolved = p.resolve_value(original)
92
+ assert resolved == {"src": "/data/in/file.csv", "dst": "/data/out/result.csv"}
93
+ assert p.tokenize_value(resolved) == original
94
+
95
+ def test_windows_backslash_in_abs_normalised(self):
96
+ p = EnvPaths([("data", r"C:\Data\proj")])
97
+ assert p.resolve_value("$data/sub") == "C:/Data/proj/sub"
98
+
99
+
100
+ # --- from env vars ---
101
+
102
+ class TestFromEnv:
103
+ def test_reads_dpe_input(self, monkeypatch):
104
+ monkeypatch.setenv("DPE_INPUT", "/mnt/input")
105
+ monkeypatch.delenv("DPE_OUTPUT", raising=False)
106
+ p = EnvPaths()
107
+ assert p.resolve_value("$input/file.csv") == "/mnt/input/file.csv"
108
+ assert p.resolve_value("$output/x") == "$output/x" # not set
109
+
110
+ def test_missing_vars_empty(self, monkeypatch):
111
+ for v in ["DPE_INPUT", "DPE_OUTPUT", "DPE_CONFIGS",
112
+ "DPE_STORAGE", "DPE_TEMP", "DPE_SESSION"]:
113
+ monkeypatch.delenv(v, raising=False)
114
+ p = EnvPaths()
115
+ assert p.is_empty()
116
+ assert p.resolve_value("$input/foo") == "$input/foo"
117
+
118
+
119
+ # --- named factories (mirror Rust SDK) ---
120
+
121
+ class TestNamedFactories:
122
+ def test_from_env_reads_dpe_vars(self, monkeypatch):
123
+ monkeypatch.setenv("DPE_INPUT", "/mnt/x")
124
+ monkeypatch.delenv("DPE_OUTPUT", raising=False)
125
+ p = EnvPaths.from_env()
126
+ assert p.resolve_value("$input/file") == "/mnt/x/file"
127
+
128
+ def test_from_pairs_bypasses_env(self, monkeypatch):
129
+ monkeypatch.setenv("DPE_INPUT", "/ignored")
130
+ p = EnvPaths.from_pairs([("input", "/explicit")])
131
+ assert p.resolve_value("$input/file") == "/explicit/file"
@@ -20,7 +20,9 @@ import pytest
20
20
 
21
21
  from dpe import _envelope as env
22
22
 
23
- SCHEMA_PATH = Path(__file__).resolve().parents[3] / "runner" / "schemas" / "stderr-events.schema.json"
23
+ SCHEMA_PATH = (
24
+ Path(__file__).resolve().parents[3] / "runner" / "schemas" / "stderr-events.schema.json"
25
+ )
24
26
 
25
27
 
26
28
  @pytest.fixture(scope="module")
@@ -1,2 +0,0 @@
1
- {"t":"d","id":"1","src":"seed","v":{"msg":"hello"}}
2
- {"t":"d","id":"2","src":"seed","v":{"msg":"world"}}
@@ -1,2 +0,0 @@
1
- {"t":"d","id":"1","src":"seed","v":{"msg":"hello"}}
2
- {"t":"d","id":"2","src":"seed","v":{"msg":"world"}}