combycode-dpe 2.0.0rc3__tar.gz → 2.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/PKG-INFO +1 -1
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/README.md +3 -1
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/pyproject.toml +1 -1
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/__init__.py +2 -0
- combycode_dpe-2.0.2/src/dpe/_cache.py +81 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_context.py +61 -9
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_envelope.py +27 -4
- combycode_dpe-2.0.2/src/dpe/_paths.py +108 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_runtime.py +25 -5
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/pyproject.toml +1 -1
- combycode_dpe-2.0.2/tests/test_cache.py +175 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/tests/test_integration.py +97 -0
- combycode_dpe-2.0.2/tests/test_paths.py +131 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/tests/test_stderr_events_conformance.py +3 -1
- combycode_dpe-2.0.0rc3/template/verify/case-basic/expected.ndjson +0 -2
- combycode_dpe-2.0.0rc3/template/verify/case-basic/input.ndjson +0 -2
- combycode_dpe-2.0.0rc3/template/verify/case-basic/settings.json +0 -1
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/.gitignore +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/src/dpe/_accumulators.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/meta.json +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/spec.yaml +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/src/__PKG__/__init__.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/src/__PKG__/main.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/template/tests/test_basic.py +0 -0
- {combycode_dpe-2.0.0rc3 → combycode_dpe-2.0.2}/tests/test_accumulators.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: combycode-dpe
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.2
|
|
4
4
|
Summary: DPE framework for building streaming pipeline tools in Python
|
|
5
5
|
Project-URL: Repository, https://github.com/combycode/dpe-framework-python
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -91,9 +91,11 @@ dpe-dev scaffold --name my-tool --runtime python --out ./my-tool --description "
|
|
|
91
91
|
cd my-tool
|
|
92
92
|
dpe-dev build . # creates .venv, installs the tool with [dev] extras
|
|
93
93
|
dpe-dev test . # pytest inside the venv
|
|
94
|
-
dpe-dev verify . # spawn the tool's main.py, feed input.ndjson, diff stdout vs expected.ndjson
|
|
95
94
|
```
|
|
96
95
|
|
|
96
|
+
End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
|
|
97
|
+
tool is wired into a pipeline — see `docs/testing.md`.
|
|
98
|
+
|
|
97
99
|
## Optional speed-up
|
|
98
100
|
|
|
99
101
|
For bulk-hashing payloads, install `xxhash` and pass `algorithm="xxhash"`
|
|
@@ -36,6 +36,7 @@ from dpe._accumulators import (
|
|
|
36
36
|
)
|
|
37
37
|
from dpe._context import Context
|
|
38
38
|
from dpe._envelope import hash_file, hash_string
|
|
39
|
+
from dpe._paths import EnvPaths
|
|
39
40
|
from dpe._runtime import Runtime
|
|
40
41
|
|
|
41
42
|
|
|
@@ -63,6 +64,7 @@ __all__ = [
|
|
|
63
64
|
"hash_string",
|
|
64
65
|
"hash_file",
|
|
65
66
|
"Context",
|
|
67
|
+
"EnvPaths",
|
|
66
68
|
"Memory",
|
|
67
69
|
"Number",
|
|
68
70
|
"Average",
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Cache helper used by Context.cached(). Mirrors the TS framework's
|
|
2
|
+
cache.ts behavior — same DPE_CACHE_MODE semantics, same on-disk layout
|
|
3
|
+
($DPE_STORAGE/<namespace>/<hash>.json), same canonical-JSON key
|
|
4
|
+
hashing.
|
|
5
|
+
|
|
6
|
+
Failure modes (cache-disabling, NOT errors propagated to user):
|
|
7
|
+
- DPE_STORAGE not set -> cache disabled, every call produces
|
|
8
|
+
- cache file unreadable -> treat as miss, log warn
|
|
9
|
+
- cache file unparseable -> treat as miss, log warn
|
|
10
|
+
- producer raises -> re-raise to caller (no cache write)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from typing import Any, TypeVar
|
|
20
|
+
|
|
21
|
+
from dpe._envelope import write_log
|
|
22
|
+
|
|
23
|
+
T = TypeVar("T")
|
|
24
|
+
|
|
25
|
+
CACHE_MODES = ("use", "refresh", "bypass", "off")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def read_cache_mode() -> str:
|
|
29
|
+
"""Read DPE_CACHE_MODE; default to "use". Unrecognized -> "use"."""
|
|
30
|
+
v = os.getenv("DPE_CACHE_MODE", "use")
|
|
31
|
+
return v if v in CACHE_MODES else "use"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def cache_path(namespace: str, key: Any) -> str | None:
|
|
35
|
+
"""Compute the on-disk path for (namespace, key). Returns None when
|
|
36
|
+
DPE_STORAGE isn't set -- caller treats that as cache disabled.
|
|
37
|
+
"""
|
|
38
|
+
storage = os.getenv("DPE_STORAGE")
|
|
39
|
+
if not storage:
|
|
40
|
+
return None
|
|
41
|
+
canonical = json.dumps(key, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
42
|
+
h = hashlib.blake2b(canonical.encode("utf-8")).hexdigest()[:32]
|
|
43
|
+
return os.path.join(storage, namespace, f"{h}.json")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def cached_impl(
|
|
47
|
+
namespace: str,
|
|
48
|
+
key: Any,
|
|
49
|
+
produce: Callable[[], T],
|
|
50
|
+
) -> T:
|
|
51
|
+
"""Read-or-produce cache around `produce`. Honors the four cache
|
|
52
|
+
modes. Returns the produced or cached value."""
|
|
53
|
+
mode = read_cache_mode()
|
|
54
|
+
path = cache_path(namespace, key)
|
|
55
|
+
|
|
56
|
+
can_read = path is not None and mode in ("use", "refresh")
|
|
57
|
+
can_write = path is not None and mode not in ("bypass", "off")
|
|
58
|
+
will_read = can_read and mode != "refresh"
|
|
59
|
+
|
|
60
|
+
if will_read and path is not None and os.path.isfile(path):
|
|
61
|
+
try:
|
|
62
|
+
with open(path, encoding="utf-8") as f:
|
|
63
|
+
value = json.load(f)
|
|
64
|
+
write_log(f"cached: hit ({namespace})", "debug")
|
|
65
|
+
return value
|
|
66
|
+
except (OSError, json.JSONDecodeError) as e:
|
|
67
|
+
write_log(f"cached: read failed ({namespace}) — {e}", "warn")
|
|
68
|
+
# Fall through to produce.
|
|
69
|
+
|
|
70
|
+
result = produce()
|
|
71
|
+
|
|
72
|
+
if can_write and path is not None:
|
|
73
|
+
try:
|
|
74
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
75
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
76
|
+
json.dump(result, f, ensure_ascii=False, separators=(",", ":"))
|
|
77
|
+
except OSError as e:
|
|
78
|
+
write_log(f"cached: write failed ({namespace}) — {e}", "warn")
|
|
79
|
+
# Don't fail the caller — they got their value.
|
|
80
|
+
|
|
81
|
+
return result
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
"""Context object — passed to every processor invocation."""
|
|
2
2
|
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any, TypeVar
|
|
5
|
+
|
|
3
6
|
from dpe._accumulators import Memory
|
|
7
|
+
from dpe._cache import cached_impl
|
|
4
8
|
from dpe._envelope import (
|
|
5
9
|
hash_file,
|
|
6
10
|
hash_string,
|
|
@@ -11,6 +15,9 @@ from dpe._envelope import (
|
|
|
11
15
|
write_stats,
|
|
12
16
|
write_trace,
|
|
13
17
|
)
|
|
18
|
+
from dpe._paths import EnvPaths
|
|
19
|
+
|
|
20
|
+
T = TypeVar("T")
|
|
14
21
|
|
|
15
22
|
|
|
16
23
|
class Context:
|
|
@@ -21,13 +28,15 @@ class Context:
|
|
|
21
28
|
Memory is shared across all processors in the tool.
|
|
22
29
|
"""
|
|
23
30
|
|
|
24
|
-
__slots__ = ("id", "src", "memory", "_runtime", "_labels")
|
|
31
|
+
__slots__ = ("id", "src", "memory", "_runtime", "_paths", "_labels")
|
|
25
32
|
|
|
26
|
-
def __init__(self, id: str, src: str, memory: Memory, runtime
|
|
33
|
+
def __init__(self, id: str, src: str, memory: Memory, runtime,
|
|
34
|
+
paths: EnvPaths | None = None):
|
|
27
35
|
self.id = id
|
|
28
36
|
self.src = src
|
|
29
37
|
self.memory = memory
|
|
30
38
|
self._runtime = runtime
|
|
39
|
+
self._paths = paths if paths is not None else EnvPaths()
|
|
31
40
|
# Labels accumulated by ctx.trace(k, v). Flushed as one merged
|
|
32
41
|
# {type:"trace"} stderr event before each ctx.output(), then cleared.
|
|
33
42
|
self._labels: dict = {}
|
|
@@ -50,15 +59,17 @@ class Context:
|
|
|
50
59
|
def output(self, v, *, id: str | None = None, src: str | None = None):
|
|
51
60
|
"""Emit data record to stdout.
|
|
52
61
|
|
|
53
|
-
Flushes accumulated trace labels as a merged trace event first
|
|
54
|
-
|
|
62
|
+
Flushes accumulated trace labels as a merged trace event first
|
|
63
|
+
(channel="data" -> counts as rows_out at the runner), then writes
|
|
64
|
+
the envelope, then clears the label bag.
|
|
65
|
+
Absolute paths in v are reverse-tokenized to $token/... form.
|
|
55
66
|
"""
|
|
56
67
|
out_id = id if id is not None else self.id
|
|
57
68
|
out_src = src if src is not None else self.src
|
|
58
|
-
# Emit trace even with empty labels
|
|
59
|
-
write_trace(out_id, out_src, self._labels)
|
|
69
|
+
# Emit trace even with empty labels -- the chain row itself is the value.
|
|
70
|
+
write_trace(out_id, out_src, self._labels, channel="data")
|
|
60
71
|
self._labels = {}
|
|
61
|
-
write_data(v, out_id, out_src)
|
|
72
|
+
write_data(self._paths.tokenize_value(v), out_id, out_src)
|
|
62
73
|
|
|
63
74
|
def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
|
|
64
75
|
"""Emit to internal named queue. Processed by process_<queue>().
|
|
@@ -82,8 +93,14 @@ class Context:
|
|
|
82
93
|
self._runtime.drain_queue()
|
|
83
94
|
|
|
84
95
|
def meta(self, v: dict):
|
|
85
|
-
"""Emit metadata record to stdout.
|
|
86
|
-
|
|
96
|
+
"""Emit metadata record to stdout.
|
|
97
|
+
|
|
98
|
+
Also emits a {type:"trace", channel:"meta"} stderr event so the
|
|
99
|
+
runner can increment its per-stage `meta` counter. Inherits
|
|
100
|
+
ctx id/src. Absolute paths in v are reverse-tokenized to $token/... form.
|
|
101
|
+
"""
|
|
102
|
+
write_trace(self.id, self.src, {}, channel="meta")
|
|
103
|
+
write_meta(self._paths.tokenize_value(v))
|
|
87
104
|
|
|
88
105
|
def log(self, msg: str, *, level: str = "info", **extra):
|
|
89
106
|
"""Write structured log to stderr."""
|
|
@@ -100,3 +117,38 @@ class Context:
|
|
|
100
117
|
def hash_file(self, filepath: str, algorithm: str = "blake2b") -> str | None:
|
|
101
118
|
"""Hash file content in chunks. Returns hex string or None on error."""
|
|
102
119
|
return hash_file(filepath, algorithm)
|
|
120
|
+
|
|
121
|
+
def cached(
|
|
122
|
+
self,
|
|
123
|
+
namespace: str,
|
|
124
|
+
key: Any,
|
|
125
|
+
produce: Callable[[], T],
|
|
126
|
+
) -> T:
|
|
127
|
+
"""Cache the result of `produce()` under
|
|
128
|
+
`$DPE_STORAGE/<namespace>/<hash>.json`.
|
|
129
|
+
|
|
130
|
+
Honors the runner's `DPE_CACHE_MODE` env:
|
|
131
|
+
- use (default) -- read cache if present, else produce + write
|
|
132
|
+
- refresh -- always produce, overwrite cache
|
|
133
|
+
- bypass -- produce, skip both read and write
|
|
134
|
+
- off -- same as bypass
|
|
135
|
+
|
|
136
|
+
`key` is canonical-JSON-hashed (blake2b, first 32 hex chars).
|
|
137
|
+
Compose it from whatever determines output equivalence.
|
|
138
|
+
|
|
139
|
+
If `$DPE_STORAGE` isn't set (e.g. tool invoked outside a
|
|
140
|
+
pipeline), cache is silently disabled — every call produces.
|
|
141
|
+
|
|
142
|
+
Producer errors propagate; failed runs do NOT poison the cache.
|
|
143
|
+
|
|
144
|
+
Example:
|
|
145
|
+
result = ctx.cached(
|
|
146
|
+
"doc-converter",
|
|
147
|
+
{"file_hash": ctx.hash_file(v["path"]),
|
|
148
|
+
"settings": settings,
|
|
149
|
+
"page": page_idx},
|
|
150
|
+
lambda: provider.convert_page(...),
|
|
151
|
+
)
|
|
152
|
+
ctx.output(result)
|
|
153
|
+
"""
|
|
154
|
+
return cached_impl(namespace, key, produce)
|
|
@@ -26,7 +26,12 @@ def hash_file(filepath: str, algorithm: str = "blake2b", chunk_size: int = 65536
|
|
|
26
26
|
try:
|
|
27
27
|
if algorithm == "xxhash":
|
|
28
28
|
try:
|
|
29
|
-
|
|
29
|
+
# xxhash is an optional runtime dep (see fallback). Need
|
|
30
|
+
# both codes here: `import-not-found` fires when xxhash isn't
|
|
31
|
+
# installed (CI), `unused-ignore` fires when it IS installed
|
|
32
|
+
# (some dev machines). Listing both keeps mypy happy in both
|
|
33
|
+
# environments without disabling the warning globally.
|
|
34
|
+
import xxhash # type: ignore[import-not-found,unused-ignore]
|
|
30
35
|
h = xxhash.xxh3_64()
|
|
31
36
|
except ImportError:
|
|
32
37
|
h = hashlib.blake2b(digest_size=16)
|
|
@@ -89,9 +94,27 @@ def write_error(v, err, id: str, src: str):
|
|
|
89
94
|
_write_stderr(record)
|
|
90
95
|
|
|
91
96
|
|
|
92
|
-
def write_trace(id: str, src: str, labels: dict):
|
|
93
|
-
"""Write merged trace event to stderr. Called by ctx before each output.
|
|
94
|
-
|
|
97
|
+
def write_trace(id: str, src: str, labels: dict, channel: str | None = None):
|
|
98
|
+
"""Write merged trace event to stderr. Called by ctx before each output.
|
|
99
|
+
|
|
100
|
+
`channel` distinguishes counter buckets at the runner side:
|
|
101
|
+
- "data" from ctx.output -> rows_out
|
|
102
|
+
- "meta" from ctx.meta -> meta
|
|
103
|
+
None is written without a `channel` key (back-compat: runner treats
|
|
104
|
+
unset as data).
|
|
105
|
+
"""
|
|
106
|
+
record: dict = {"type": "trace", "id": id, "src": src, "labels": labels}
|
|
107
|
+
if channel is not None:
|
|
108
|
+
record["channel"] = channel
|
|
109
|
+
_write_stderr(record)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def write_input(id: str, src: str):
|
|
113
|
+
"""Write input event to stderr. Emitted by the runtime per envelope read
|
|
114
|
+
from stdin, BEFORE the processor runs. Lets the runner count rows_in
|
|
115
|
+
for stages that never call ctx.output() (pass-through, terminal sinks).
|
|
116
|
+
"""
|
|
117
|
+
_write_stderr({"type": "input", "id": id, "src": src})
|
|
95
118
|
|
|
96
119
|
|
|
97
120
|
def write_stats(data: dict):
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""EnvPaths — resolves $token path prefixes in envelope v on input;
|
|
2
|
+
reverse-tokenizes absolute paths back to $token form on output.
|
|
3
|
+
|
|
4
|
+
Input side (runtime, before processor): $token/subpath -> absolute path
|
|
5
|
+
Output side (context.output / context.meta): absolute path -> $token/subpath
|
|
6
|
+
|
|
7
|
+
Both sides are a no-op when the DPE_* env vars are not set.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
_TOKEN_MAP = [
|
|
14
|
+
("input", "DPE_INPUT"),
|
|
15
|
+
("output", "DPE_OUTPUT"),
|
|
16
|
+
("configs", "DPE_CONFIGS"),
|
|
17
|
+
("storage", "DPE_STORAGE"),
|
|
18
|
+
("temp", "DPE_TEMP"),
|
|
19
|
+
("session", "DPE_SESSION"),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class EnvPaths:
|
|
24
|
+
"""Resolved env prefix map. Build once at tool startup."""
|
|
25
|
+
|
|
26
|
+
__slots__ = ("_entries",)
|
|
27
|
+
|
|
28
|
+
def __init__(self, pairs: list[tuple[str, str]] | None = None):
|
|
29
|
+
"""
|
|
30
|
+
If *pairs* is None the standard DPE_* env vars are read.
|
|
31
|
+
Pass explicit pairs for tests: ``[("input", "/abs/input"), ...]``.
|
|
32
|
+
|
|
33
|
+
Prefer the named factories :meth:`from_env` and :meth:`from_pairs`
|
|
34
|
+
-- they're consistent with the Rust SDK (``from_env()`` /
|
|
35
|
+
``from_pairs(&[...])``) and read more clearly at call sites. The
|
|
36
|
+
polymorphic constructor is kept for backwards compat.
|
|
37
|
+
"""
|
|
38
|
+
if pairs is None:
|
|
39
|
+
raw = []
|
|
40
|
+
for token, env_var in _TOKEN_MAP:
|
|
41
|
+
val = os.environ.get(env_var, "")
|
|
42
|
+
if val:
|
|
43
|
+
raw.append((token, val.replace("\\", "/")))
|
|
44
|
+
else:
|
|
45
|
+
raw = [(t, a.replace("\\", "/")) for t, a in pairs]
|
|
46
|
+
# Sort longest-abs-first for greedy tokenization.
|
|
47
|
+
self._entries: list[tuple[str, str]] = sorted(
|
|
48
|
+
raw, key=lambda e: len(e[1]), reverse=True
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def from_env(cls) -> "EnvPaths":
|
|
53
|
+
"""Read the standard DPE_* env vars; missing/empty vars are skipped."""
|
|
54
|
+
return cls()
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_pairs(cls, pairs: list[tuple[str, str]]) -> "EnvPaths":
|
|
58
|
+
"""Construct from explicit (token, abs_path) pairs -- useful for tests."""
|
|
59
|
+
return cls(pairs)
|
|
60
|
+
|
|
61
|
+
def is_empty(self) -> bool:
|
|
62
|
+
return not self._entries
|
|
63
|
+
|
|
64
|
+
def resolve_value(self, v: Any) -> Any:
|
|
65
|
+
"""Walk v, resolving $token[/subpath] strings to absolute paths."""
|
|
66
|
+
if self.is_empty():
|
|
67
|
+
return v
|
|
68
|
+
return self._walk(v, self._resolve_str)
|
|
69
|
+
|
|
70
|
+
def tokenize_value(self, v: Any) -> Any:
|
|
71
|
+
"""Walk v, tokenizing absolute paths back to $token[/subpath] form."""
|
|
72
|
+
if self.is_empty():
|
|
73
|
+
return v
|
|
74
|
+
return self._walk(v, self._tokenize_str)
|
|
75
|
+
|
|
76
|
+
def _walk(self, v: Any, f) -> Any:
|
|
77
|
+
if isinstance(v, str):
|
|
78
|
+
return f(v)
|
|
79
|
+
if isinstance(v, list):
|
|
80
|
+
return [self._walk(x, f) for x in v]
|
|
81
|
+
if isinstance(v, dict):
|
|
82
|
+
return {k: self._walk(val, f) for k, val in v.items()}
|
|
83
|
+
return v
|
|
84
|
+
|
|
85
|
+
def _resolve_str(self, s: str) -> str:
|
|
86
|
+
if not s.startswith("$"):
|
|
87
|
+
return s
|
|
88
|
+
slash = s.find("/")
|
|
89
|
+
if slash < 0:
|
|
90
|
+
name, tail = s[1:], ""
|
|
91
|
+
else:
|
|
92
|
+
name, tail = s[1:slash], s[slash:]
|
|
93
|
+
for token, abs_path in self._entries:
|
|
94
|
+
if token == name:
|
|
95
|
+
return abs_path + tail if tail else abs_path
|
|
96
|
+
return s
|
|
97
|
+
|
|
98
|
+
def _tokenize_str(self, s: str) -> str:
|
|
99
|
+
normalized = s.replace("\\", "/")
|
|
100
|
+
for token, abs_path in self._entries:
|
|
101
|
+
if normalized.startswith(abs_path):
|
|
102
|
+
rest = normalized[len(abs_path):]
|
|
103
|
+
if not rest:
|
|
104
|
+
return f"${token}"
|
|
105
|
+
if rest.startswith("/"):
|
|
106
|
+
return f"${token}{rest}"
|
|
107
|
+
# rest doesn't start with "/" -- partial component match, skip
|
|
108
|
+
return s
|
|
@@ -9,7 +9,8 @@ from collections.abc import Callable
|
|
|
9
9
|
|
|
10
10
|
from dpe._accumulators import Memory
|
|
11
11
|
from dpe._context import Context
|
|
12
|
-
from dpe._envelope import parse_envelope, write_log
|
|
12
|
+
from dpe._envelope import parse_envelope, write_input, write_log
|
|
13
|
+
from dpe._paths import EnvPaths
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class Runtime:
|
|
@@ -22,6 +23,7 @@ class Runtime:
|
|
|
22
23
|
self.processors: dict[str, Callable] = {}
|
|
23
24
|
self._queue: deque = deque()
|
|
24
25
|
self._shutdown = False
|
|
26
|
+
self._paths = EnvPaths()
|
|
25
27
|
|
|
26
28
|
def enqueue(self, queue_name: str, v, id: str, src: str):
|
|
27
29
|
"""Add item to internal queue."""
|
|
@@ -40,7 +42,7 @@ class Runtime:
|
|
|
40
42
|
write_log(f"No processor for queue '{name}', dropping item", "warn")
|
|
41
43
|
continue
|
|
42
44
|
|
|
43
|
-
ctx = Context(item_id, item_src, self.memory, self)
|
|
45
|
+
ctx = Context(item_id, item_src, self.memory, self, self._paths)
|
|
44
46
|
try:
|
|
45
47
|
proc(v, self.settings, ctx)
|
|
46
48
|
except Exception as e:
|
|
@@ -52,7 +54,7 @@ class Runtime:
|
|
|
52
54
|
write_log(f"Queue drain hit safety limit ({max_iterations})", "error")
|
|
53
55
|
|
|
54
56
|
def create_ctx(self, id: str, src: str) -> Context:
|
|
55
|
-
return Context(id, src, self.memory, self)
|
|
57
|
+
return Context(id, src, self.memory, self, self._paths)
|
|
56
58
|
|
|
57
59
|
def _handle_signal(self, signum, frame):
|
|
58
60
|
self._shutdown = True
|
|
@@ -71,6 +73,14 @@ class Runtime:
|
|
|
71
73
|
|
|
72
74
|
# Parse settings
|
|
73
75
|
self.settings = _parse_settings()
|
|
76
|
+
# `accept_meta: true` in settings → the read loop dispatches t:"m"
|
|
77
|
+
# envelopes to process_input alongside t:"d" (instead of skipping).
|
|
78
|
+
# Default false preserves historical behavior — meta envelopes only
|
|
79
|
+
# matter to tools that explicitly opt in (typically sinks like
|
|
80
|
+
# write-file-stream when used as a meta-output target).
|
|
81
|
+
accept_meta = bool(
|
|
82
|
+
isinstance(self.settings, dict) and self.settings.get("accept_meta") is True
|
|
83
|
+
)
|
|
74
84
|
|
|
75
85
|
# Main stdin loop
|
|
76
86
|
for line in sys.stdin:
|
|
@@ -81,13 +91,23 @@ class Runtime:
|
|
|
81
91
|
if envelope is None:
|
|
82
92
|
continue
|
|
83
93
|
|
|
84
|
-
#
|
|
85
|
-
|
|
94
|
+
# Default: only data envelopes. With accept_meta=True, also
|
|
95
|
+
# accept t:"m". Anything else (unknown / missing t) skipped.
|
|
96
|
+
t = envelope.get("t")
|
|
97
|
+
if t != "d" and not (accept_meta and t == "m"):
|
|
86
98
|
continue
|
|
87
99
|
|
|
88
100
|
id = envelope.get("id", "")
|
|
89
101
|
src = envelope.get("src", "")
|
|
90
102
|
v = envelope.get("v", {})
|
|
103
|
+
# Resolve $token paths in v before handing to processor.
|
|
104
|
+
v = self._paths.resolve_value(v)
|
|
105
|
+
|
|
106
|
+
# Emit an `input` event BEFORE the processor runs so the runner
|
|
107
|
+
# can count rows_in for every stage that reads stdin — including
|
|
108
|
+
# pass-through tools and terminal sinks that never call
|
|
109
|
+
# ctx.output() (and therefore wouldn't otherwise emit a trace).
|
|
110
|
+
write_input(id, src)
|
|
91
111
|
|
|
92
112
|
ctx = self.create_ctx(id, src)
|
|
93
113
|
try:
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Tests for the cache helper. Mirror coverage of the TS suite."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import tempfile
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
11
|
+
from dpe._cache import cache_path, cached_impl, read_cache_mode
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@pytest.fixture
|
|
15
|
+
def storage_dir(monkeypatch):
|
|
16
|
+
"""Set DPE_STORAGE to a tempdir for the test, clear DPE_CACHE_MODE."""
|
|
17
|
+
with tempfile.TemporaryDirectory(prefix="dpe-cache-test-") as d:
|
|
18
|
+
monkeypatch.setenv("DPE_STORAGE", d)
|
|
19
|
+
monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
|
|
20
|
+
yield d
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ─── read_cache_mode ─────────────────────────────────────────────────────
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_read_cache_mode_default(monkeypatch):
|
|
27
|
+
monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
|
|
28
|
+
assert read_cache_mode() == "use"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize("m", ["use", "refresh", "bypass", "off"])
|
|
32
|
+
def test_read_cache_mode_recognized(monkeypatch, m):
|
|
33
|
+
monkeypatch.setenv("DPE_CACHE_MODE", m)
|
|
34
|
+
assert read_cache_mode() == m
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_read_cache_mode_garbage(monkeypatch):
|
|
38
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "nonsense")
|
|
39
|
+
assert read_cache_mode() == "use"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ─── cache_path ──────────────────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_cache_path_none_when_no_storage(monkeypatch):
|
|
46
|
+
monkeypatch.delenv("DPE_STORAGE", raising=False)
|
|
47
|
+
assert cache_path("ns", {"k": 1}) is None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_cache_path_key_order_stable(storage_dir):
|
|
51
|
+
a = cache_path("ns", {"k": 1, "m": 2})
|
|
52
|
+
b = cache_path("ns", {"m": 2, "k": 1})
|
|
53
|
+
assert a == b
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_cache_path_different_keys(storage_dir):
|
|
57
|
+
assert cache_path("ns", {"k": 1}) != cache_path("ns", {"k": 2})
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_cache_path_different_namespaces(storage_dir):
|
|
61
|
+
assert cache_path("ns1", {"k": 1}) != cache_path("ns2", {"k": 1})
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ─── cached_impl ─────────────────────────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_miss_calls_produce_writes_file(storage_dir):
|
|
68
|
+
calls = [0]
|
|
69
|
+
|
|
70
|
+
def produce():
|
|
71
|
+
calls[0] += 1
|
|
72
|
+
return {"hello": "world"}
|
|
73
|
+
|
|
74
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
75
|
+
assert result == {"hello": "world"}
|
|
76
|
+
assert calls[0] == 1
|
|
77
|
+
assert Path(cache_path("ns", {"k": 1})).is_file()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_hit_skips_produce(storage_dir):
|
|
81
|
+
# Prime via miss to create the dir.
|
|
82
|
+
cached_impl("ns", {"k": 1}, lambda: {"initial": True})
|
|
83
|
+
# Overwrite with our seeded value.
|
|
84
|
+
Path(cache_path("ns", {"k": 1})).write_text(json.dumps({"cached": True}))
|
|
85
|
+
|
|
86
|
+
calls = [0]
|
|
87
|
+
|
|
88
|
+
def produce():
|
|
89
|
+
calls[0] += 1
|
|
90
|
+
return {"fresh": True}
|
|
91
|
+
|
|
92
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
93
|
+
assert result == {"cached": True}
|
|
94
|
+
assert calls[0] == 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_refresh_always_produces_overwrites(storage_dir, monkeypatch):
|
|
98
|
+
cached_impl("ns", {"k": 1}, lambda: {"first": True})
|
|
99
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "refresh")
|
|
100
|
+
|
|
101
|
+
calls = [0]
|
|
102
|
+
|
|
103
|
+
def produce():
|
|
104
|
+
calls[0] += 1
|
|
105
|
+
return {"fresh": True}
|
|
106
|
+
|
|
107
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
108
|
+
assert result == {"fresh": True}
|
|
109
|
+
assert calls[0] == 1
|
|
110
|
+
on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
|
|
111
|
+
assert on_disk == {"fresh": True}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_bypass_no_read_no_write(storage_dir, monkeypatch):
|
|
115
|
+
cached_impl("ns", {"k": 1}, lambda: {"initial": True})
|
|
116
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "bypass")
|
|
117
|
+
|
|
118
|
+
result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
|
|
119
|
+
assert result == {"fresh": True}
|
|
120
|
+
on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
|
|
121
|
+
# bypass does NOT write — original cache content unchanged.
|
|
122
|
+
assert on_disk == {"initial": True}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_off_same_as_bypass(storage_dir, monkeypatch):
|
|
126
|
+
cached_impl("ns", {"k": 1}, lambda: {"initial": True})
|
|
127
|
+
monkeypatch.setenv("DPE_CACHE_MODE", "off")
|
|
128
|
+
result = cached_impl("ns", {"k": 1}, lambda: {"fresh": True})
|
|
129
|
+
assert result == {"fresh": True}
|
|
130
|
+
on_disk = json.loads(Path(cache_path("ns", {"k": 1})).read_text())
|
|
131
|
+
assert on_disk == {"initial": True}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_no_storage_disables_cache(monkeypatch):
|
|
135
|
+
monkeypatch.delenv("DPE_STORAGE", raising=False)
|
|
136
|
+
monkeypatch.delenv("DPE_CACHE_MODE", raising=False)
|
|
137
|
+
|
|
138
|
+
calls = [0]
|
|
139
|
+
|
|
140
|
+
def produce():
|
|
141
|
+
calls[0] += 1
|
|
142
|
+
return {"n": calls[0]}
|
|
143
|
+
|
|
144
|
+
r1 = cached_impl("ns", {"k": 1}, produce)
|
|
145
|
+
r2 = cached_impl("ns", {"k": 1}, produce)
|
|
146
|
+
assert r1 == {"n": 1}
|
|
147
|
+
assert r2 == {"n": 2}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_producer_error_propagates_no_write(storage_dir):
|
|
151
|
+
attempts = [0]
|
|
152
|
+
|
|
153
|
+
def boom():
|
|
154
|
+
attempts[0] += 1
|
|
155
|
+
raise RuntimeError("kaboom")
|
|
156
|
+
|
|
157
|
+
with pytest.raises(RuntimeError, match="kaboom"):
|
|
158
|
+
cached_impl("ns", {"k": 1}, boom)
|
|
159
|
+
assert attempts[0] == 1
|
|
160
|
+
assert not Path(cache_path("ns", {"k": 1})).is_file()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_malformed_cache_treated_as_miss(storage_dir):
|
|
164
|
+
cached_impl("ns", {"k": 1}, lambda: {"ok": True})
|
|
165
|
+
Path(cache_path("ns", {"k": 1})).write_text("this is not json")
|
|
166
|
+
|
|
167
|
+
calls = [0]
|
|
168
|
+
|
|
169
|
+
def produce():
|
|
170
|
+
calls[0] += 1
|
|
171
|
+
return {"recovered": True}
|
|
172
|
+
|
|
173
|
+
result = cached_impl("ns", {"k": 1}, produce)
|
|
174
|
+
assert result == {"recovered": True}
|
|
175
|
+
assert calls[0] == 1
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Integration tests — full tool lifecycle via subprocess."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
import os
|
|
4
5
|
import subprocess
|
|
5
6
|
import sys
|
|
6
7
|
import tempfile
|
|
@@ -546,3 +547,99 @@ class TestContext:
|
|
|
546
547
|
|
|
547
548
|
assert data[0]["v"]["id"] == "my-id"
|
|
548
549
|
assert data[0]["v"]["src"] == "my-src"
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
class TestEnvPathsRoundTrip:
|
|
553
|
+
def _run_with_env(self, code, settings, stdin_lines, env_overrides):
|
|
554
|
+
"""Like run_tool_code but injects custom DPE_* env vars into the subprocess."""
|
|
555
|
+
import subprocess
|
|
556
|
+
import sys
|
|
557
|
+
import tempfile
|
|
558
|
+
import textwrap
|
|
559
|
+
from pathlib import Path
|
|
560
|
+
|
|
561
|
+
stdin_data = (
|
|
562
|
+
"\n".join(json.dumps(r) for r in stdin_lines) + "\n" if stdin_lines else ""
|
|
563
|
+
)
|
|
564
|
+
with tempfile.NamedTemporaryFile(
|
|
565
|
+
mode="w", suffix=".py", delete=False,
|
|
566
|
+
dir=tempfile.gettempdir(), encoding="utf-8",
|
|
567
|
+
) as f:
|
|
568
|
+
f.write(textwrap.dedent(code))
|
|
569
|
+
f.flush()
|
|
570
|
+
tool_path = f.name
|
|
571
|
+
|
|
572
|
+
env = {**os.environ, **env_overrides}
|
|
573
|
+
result = subprocess.run(
|
|
574
|
+
[sys.executable, tool_path, json.dumps(settings)],
|
|
575
|
+
input=stdin_data, capture_output=True, text=True,
|
|
576
|
+
timeout=30, env=env,
|
|
577
|
+
)
|
|
578
|
+
Path(tool_path).unlink(missing_ok=True)
|
|
579
|
+
|
|
580
|
+
data = []
|
|
581
|
+
for line in result.stdout.strip().split("\n"):
|
|
582
|
+
if line:
|
|
583
|
+
rec = json.loads(line)
|
|
584
|
+
if rec.get("t") == "d":
|
|
585
|
+
data.append(rec)
|
|
586
|
+
return data
|
|
587
|
+
|
|
588
|
+
def test_token_resolved_on_input(self):
|
|
589
|
+
"""$input in envelope v is expanded to abs path before processor sees it.
|
|
590
|
+
|
|
591
|
+
The processor inspects the path (not outputs it) so the tokenize-on-output
|
|
592
|
+
step does not reverse the expansion — we test the expansion independently.
|
|
593
|
+
"""
|
|
594
|
+
data = self._run_with_env("""
|
|
595
|
+
import dpe
|
|
596
|
+
|
|
597
|
+
def process_input(v, settings, ctx):
|
|
598
|
+
# Check the resolved value; output a boolean, not the path itself,
|
|
599
|
+
# so the tokenize-on-output step doesn't convert it back.
|
|
600
|
+
ctx.output({"starts_abs": v["path"].startswith("/mnt/drive")})
|
|
601
|
+
|
|
602
|
+
if __name__ == "__main__":
|
|
603
|
+
dpe.run()
|
|
604
|
+
""", {}, [inp({"path": "$input/data.csv"})],
|
|
605
|
+
{"DPE_INPUT": "/mnt/drive"})
|
|
606
|
+
|
|
607
|
+
assert data[0]["v"]["starts_abs"] is True
|
|
608
|
+
|
|
609
|
+
def test_abs_path_tokenized_on_output(self):
|
|
610
|
+
"""Absolute paths in ctx.output(v) are tokenized back to $token form.
|
|
611
|
+
|
|
612
|
+
The processor hardcodes an absolute path string (not from resolved input)
|
|
613
|
+
so we're testing only the output tokenization.
|
|
614
|
+
"""
|
|
615
|
+
data = self._run_with_env("""
|
|
616
|
+
import dpe
|
|
617
|
+
|
|
618
|
+
def process_input(v, settings, ctx):
|
|
619
|
+
# Output a hard-coded abs path -- not from resolved input.
|
|
620
|
+
ctx.output({"path": "/mnt/drive/result.csv"})
|
|
621
|
+
|
|
622
|
+
if __name__ == "__main__":
|
|
623
|
+
dpe.run()
|
|
624
|
+
""", {}, [inp({"x": 1})],
|
|
625
|
+
{"DPE_INPUT": "/mnt/drive"})
|
|
626
|
+
|
|
627
|
+
assert data[0]["v"]["path"] == "$input/result.csv"
|
|
628
|
+
|
|
629
|
+
def test_no_env_passthrough(self):
|
|
630
|
+
"""Without DPE_* vars set, $token paths pass through unchanged."""
|
|
631
|
+
env_overrides = {k: "" for k in [
|
|
632
|
+
"DPE_INPUT", "DPE_OUTPUT", "DPE_CONFIGS",
|
|
633
|
+
"DPE_STORAGE", "DPE_TEMP", "DPE_SESSION",
|
|
634
|
+
]}
|
|
635
|
+
data = self._run_with_env("""
|
|
636
|
+
import dpe
|
|
637
|
+
|
|
638
|
+
def process_input(v, settings, ctx):
|
|
639
|
+
ctx.output(v)
|
|
640
|
+
|
|
641
|
+
if __name__ == "__main__":
|
|
642
|
+
dpe.run()
|
|
643
|
+
""", {}, [inp({"path": "$input/foo"})], env_overrides)
|
|
644
|
+
|
|
645
|
+
assert data[0]["v"]["path"] == "$input/foo"
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Tests for EnvPaths -- $token resolve/tokenize round-trip."""
|
|
2
|
+
|
|
3
|
+
from dpe._paths import EnvPaths
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def paths():
|
|
7
|
+
return EnvPaths([
|
|
8
|
+
("input", "/abs/input"),
|
|
9
|
+
("output", "/abs/output"),
|
|
10
|
+
("storage", "/abs/storage"),
|
|
11
|
+
# Intentionally shorter abs -- tests longest-match logic.
|
|
12
|
+
("session", "/abs/storage/session"),
|
|
13
|
+
])
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# --- resolve_value ---
|
|
17
|
+
|
|
18
|
+
class TestResolveValue:
|
|
19
|
+
def test_top_level_token(self):
|
|
20
|
+
assert paths().resolve_value("$input") == "/abs/input"
|
|
21
|
+
|
|
22
|
+
def test_token_with_subpath(self):
|
|
23
|
+
assert paths().resolve_value("$input/data/file.csv") == "/abs/input/data/file.csv"
|
|
24
|
+
|
|
25
|
+
def test_unknown_token_unchanged(self):
|
|
26
|
+
assert paths().resolve_value("$set/field") == "$set/field"
|
|
27
|
+
assert paths().resolve_value("$bogus") == "$bogus"
|
|
28
|
+
|
|
29
|
+
def test_recurses_into_dict(self):
|
|
30
|
+
v = {"path": "$input/a.csv", "n": 42, "nested": {"p": "$output/b"}}
|
|
31
|
+
r = paths().resolve_value(v)
|
|
32
|
+
assert r["path"] == "/abs/input/a.csv"
|
|
33
|
+
assert r["n"] == 42
|
|
34
|
+
assert r["nested"]["p"] == "/abs/output/b"
|
|
35
|
+
|
|
36
|
+
def test_recurses_into_list(self):
|
|
37
|
+
v = ["$input/x", "$output/y", "plain"]
|
|
38
|
+
r = paths().resolve_value(v)
|
|
39
|
+
assert r == ["/abs/input/x", "/abs/output/y", "plain"]
|
|
40
|
+
|
|
41
|
+
def test_no_env_is_noop(self):
|
|
42
|
+
empty = EnvPaths([])
|
|
43
|
+
v = {"p": "$input/foo"}
|
|
44
|
+
assert empty.resolve_value(v) == v
|
|
45
|
+
|
|
46
|
+
def test_non_string_types_unchanged(self):
|
|
47
|
+
v = {"n": 42, "b": True, "nil": None, "p": "$input"}
|
|
48
|
+
r = paths().resolve_value(v)
|
|
49
|
+
assert r["n"] == 42
|
|
50
|
+
assert r["b"] is True
|
|
51
|
+
assert r["nil"] is None
|
|
52
|
+
assert r["p"] == "/abs/input"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# --- tokenize_value ---
|
|
56
|
+
|
|
57
|
+
class TestTokenizeValue:
|
|
58
|
+
def test_exact_prefix(self):
|
|
59
|
+
assert paths().tokenize_value("/abs/input") == "$input"
|
|
60
|
+
|
|
61
|
+
def test_prefix_with_subpath(self):
|
|
62
|
+
assert paths().tokenize_value("/abs/output/results/out.csv") == "$output/results/out.csv"
|
|
63
|
+
|
|
64
|
+
def test_partial_component_not_replaced(self):
|
|
65
|
+
# "/abs/inputXYZ" must NOT match "$input"
|
|
66
|
+
assert paths().tokenize_value("/abs/inputXYZ") == "/abs/inputXYZ"
|
|
67
|
+
|
|
68
|
+
def test_longest_prefix_wins(self):
|
|
69
|
+
# "/abs/storage/session/..." should match "$session", not "$storage"
|
|
70
|
+
assert paths().tokenize_value("/abs/storage/session/data.json") == "$session/data.json"
|
|
71
|
+
|
|
72
|
+
def test_non_path_unchanged(self):
|
|
73
|
+
assert paths().tokenize_value("hello world") == "hello world"
|
|
74
|
+
assert paths().tokenize_value(42) == 42
|
|
75
|
+
assert paths().tokenize_value(True) is True
|
|
76
|
+
assert paths().tokenize_value(None) is None
|
|
77
|
+
|
|
78
|
+
def test_recurses_into_nested(self):
|
|
79
|
+
v = {"a": {"b": "/abs/input/x"}, "arr": ["/abs/output/y"]}
|
|
80
|
+
r = paths().tokenize_value(v)
|
|
81
|
+
assert r["a"]["b"] == "$input/x"
|
|
82
|
+
assert r["arr"][0] == "$output/y"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# --- round-trip ---
|
|
86
|
+
|
|
87
|
+
class TestRoundTrip:
|
|
88
|
+
def test_resolve_then_tokenize(self):
|
|
89
|
+
p = EnvPaths([("input", "/data/in"), ("output", "/data/out")])
|
|
90
|
+
original = {"src": "$input/file.csv", "dst": "$output/result.csv"}
|
|
91
|
+
resolved = p.resolve_value(original)
|
|
92
|
+
assert resolved == {"src": "/data/in/file.csv", "dst": "/data/out/result.csv"}
|
|
93
|
+
assert p.tokenize_value(resolved) == original
|
|
94
|
+
|
|
95
|
+
def test_windows_backslash_in_abs_normalised(self):
|
|
96
|
+
p = EnvPaths([("data", r"C:\Data\proj")])
|
|
97
|
+
assert p.resolve_value("$data/sub") == "C:/Data/proj/sub"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# --- from env vars ---
|
|
101
|
+
|
|
102
|
+
class TestFromEnv:
|
|
103
|
+
def test_reads_dpe_input(self, monkeypatch):
|
|
104
|
+
monkeypatch.setenv("DPE_INPUT", "/mnt/input")
|
|
105
|
+
monkeypatch.delenv("DPE_OUTPUT", raising=False)
|
|
106
|
+
p = EnvPaths()
|
|
107
|
+
assert p.resolve_value("$input/file.csv") == "/mnt/input/file.csv"
|
|
108
|
+
assert p.resolve_value("$output/x") == "$output/x" # not set
|
|
109
|
+
|
|
110
|
+
def test_missing_vars_empty(self, monkeypatch):
|
|
111
|
+
for v in ["DPE_INPUT", "DPE_OUTPUT", "DPE_CONFIGS",
|
|
112
|
+
"DPE_STORAGE", "DPE_TEMP", "DPE_SESSION"]:
|
|
113
|
+
monkeypatch.delenv(v, raising=False)
|
|
114
|
+
p = EnvPaths()
|
|
115
|
+
assert p.is_empty()
|
|
116
|
+
assert p.resolve_value("$input/foo") == "$input/foo"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# --- named factories (mirror Rust SDK) ---
|
|
120
|
+
|
|
121
|
+
class TestNamedFactories:
|
|
122
|
+
def test_from_env_reads_dpe_vars(self, monkeypatch):
|
|
123
|
+
monkeypatch.setenv("DPE_INPUT", "/mnt/x")
|
|
124
|
+
monkeypatch.delenv("DPE_OUTPUT", raising=False)
|
|
125
|
+
p = EnvPaths.from_env()
|
|
126
|
+
assert p.resolve_value("$input/file") == "/mnt/x/file"
|
|
127
|
+
|
|
128
|
+
def test_from_pairs_bypasses_env(self, monkeypatch):
|
|
129
|
+
monkeypatch.setenv("DPE_INPUT", "/ignored")
|
|
130
|
+
p = EnvPaths.from_pairs([("input", "/explicit")])
|
|
131
|
+
assert p.resolve_value("$input/file") == "/explicit/file"
|
|
@@ -20,7 +20,9 @@ import pytest
|
|
|
20
20
|
|
|
21
21
|
from dpe import _envelope as env
|
|
22
22
|
|
|
23
|
-
SCHEMA_PATH =
|
|
23
|
+
SCHEMA_PATH = (
|
|
24
|
+
Path(__file__).resolve().parents[3] / "runner" / "schemas" / "stderr-events.schema.json"
|
|
25
|
+
)
|
|
24
26
|
|
|
25
27
|
|
|
26
28
|
@pytest.fixture(scope="module")
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|