combycode-dpe 2.0.1__tar.gz → 2.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/PKG-INFO +1 -1
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/README.md +3 -1
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/pyproject.toml +1 -1
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/src/dpe/__init__.py +2 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/src/dpe/_context.py +11 -7
- combycode_dpe-2.0.2/src/dpe/_paths.py +108 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/src/dpe/_runtime.py +6 -2
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/tests/test_integration.py +97 -0
- combycode_dpe-2.0.2/tests/test_paths.py +131 -0
- combycode_dpe-2.0.1/template/verify/case-basic/expected.ndjson +0 -2
- combycode_dpe-2.0.1/template/verify/case-basic/input.ndjson +0 -2
- combycode_dpe-2.0.1/template/verify/case-basic/settings.json +0 -1
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/.gitignore +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/src/dpe/_accumulators.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/src/dpe/_cache.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/src/dpe/_envelope.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/template/meta.json +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/template/pyproject.toml +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/template/spec.yaml +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/template/src/__PKG__/__init__.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/template/src/__PKG__/main.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/template/tests/test_basic.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/tests/test_accumulators.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/tests/test_cache.py +0 -0
- {combycode_dpe-2.0.1 → combycode_dpe-2.0.2}/tests/test_stderr_events_conformance.py +0 -0
|
@@ -91,9 +91,11 @@ dpe-dev scaffold --name my-tool --runtime python --out ./my-tool --description "
|
|
|
91
91
|
cd my-tool
|
|
92
92
|
dpe-dev build . # creates .venv, installs the tool with [dev] extras
|
|
93
93
|
dpe-dev test . # pytest inside the venv
|
|
94
|
-
dpe-dev verify . # spawn the tool's main.py, feed input.ndjson, diff stdout vs expected.ndjson
|
|
95
94
|
```
|
|
96
95
|
|
|
96
|
+
End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
|
|
97
|
+
tool is wired into a pipeline — see `docs/testing.md`.
|
|
98
|
+
|
|
97
99
|
## Optional speed-up
|
|
98
100
|
|
|
99
101
|
For bulk-hashing payloads, install `xxhash` and pass `algorithm="xxhash"`
|
|
@@ -36,6 +36,7 @@ from dpe._accumulators import (
|
|
|
36
36
|
)
|
|
37
37
|
from dpe._context import Context
|
|
38
38
|
from dpe._envelope import hash_file, hash_string
|
|
39
|
+
from dpe._paths import EnvPaths
|
|
39
40
|
from dpe._runtime import Runtime
|
|
40
41
|
|
|
41
42
|
|
|
@@ -63,6 +64,7 @@ __all__ = [
|
|
|
63
64
|
"hash_string",
|
|
64
65
|
"hash_file",
|
|
65
66
|
"Context",
|
|
67
|
+
"EnvPaths",
|
|
66
68
|
"Memory",
|
|
67
69
|
"Number",
|
|
68
70
|
"Average",
|
|
@@ -15,6 +15,7 @@ from dpe._envelope import (
|
|
|
15
15
|
write_stats,
|
|
16
16
|
write_trace,
|
|
17
17
|
)
|
|
18
|
+
from dpe._paths import EnvPaths
|
|
18
19
|
|
|
19
20
|
T = TypeVar("T")
|
|
20
21
|
|
|
@@ -27,13 +28,15 @@ class Context:
|
|
|
27
28
|
Memory is shared across all processors in the tool.
|
|
28
29
|
"""
|
|
29
30
|
|
|
30
|
-
__slots__ = ("id", "src", "memory", "_runtime", "_labels")
|
|
31
|
+
__slots__ = ("id", "src", "memory", "_runtime", "_paths", "_labels")
|
|
31
32
|
|
|
32
|
-
def __init__(self, id: str, src: str, memory: Memory, runtime
|
|
33
|
+
def __init__(self, id: str, src: str, memory: Memory, runtime,
|
|
34
|
+
paths: EnvPaths | None = None):
|
|
33
35
|
self.id = id
|
|
34
36
|
self.src = src
|
|
35
37
|
self.memory = memory
|
|
36
38
|
self._runtime = runtime
|
|
39
|
+
self._paths = paths if paths is not None else EnvPaths()
|
|
37
40
|
# Labels accumulated by ctx.trace(k, v). Flushed as one merged
|
|
38
41
|
# {type:"trace"} stderr event before each ctx.output(), then cleared.
|
|
39
42
|
self._labels: dict = {}
|
|
@@ -57,15 +60,16 @@ class Context:
|
|
|
57
60
|
"""Emit data record to stdout.
|
|
58
61
|
|
|
59
62
|
Flushes accumulated trace labels as a merged trace event first
|
|
60
|
-
(channel="data"
|
|
63
|
+
(channel="data" -> counts as rows_out at the runner), then writes
|
|
61
64
|
the envelope, then clears the label bag.
|
|
65
|
+
Absolute paths in v are reverse-tokenized to $token/... form.
|
|
62
66
|
"""
|
|
63
67
|
out_id = id if id is not None else self.id
|
|
64
68
|
out_src = src if src is not None else self.src
|
|
65
|
-
# Emit trace even with empty labels
|
|
69
|
+
# Emit trace even with empty labels -- the chain row itself is the value.
|
|
66
70
|
write_trace(out_id, out_src, self._labels, channel="data")
|
|
67
71
|
self._labels = {}
|
|
68
|
-
write_data(v, out_id, out_src)
|
|
72
|
+
write_data(self._paths.tokenize_value(v), out_id, out_src)
|
|
69
73
|
|
|
70
74
|
def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
|
|
71
75
|
"""Emit to internal named queue. Processed by process_<queue>().
|
|
@@ -93,10 +97,10 @@ class Context:
|
|
|
93
97
|
|
|
94
98
|
Also emits a {type:"trace", channel:"meta"} stderr event so the
|
|
95
99
|
runner can increment its per-stage `meta` counter. Inherits
|
|
96
|
-
ctx id/src.
|
|
100
|
+
ctx id/src. Absolute paths in v are reverse-tokenized to $token/... form.
|
|
97
101
|
"""
|
|
98
102
|
write_trace(self.id, self.src, {}, channel="meta")
|
|
99
|
-
write_meta(v)
|
|
103
|
+
write_meta(self._paths.tokenize_value(v))
|
|
100
104
|
|
|
101
105
|
def log(self, msg: str, *, level: str = "info", **extra):
|
|
102
106
|
"""Write structured log to stderr."""
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""EnvPaths — resolves $token path prefixes in envelope v on input;
|
|
2
|
+
reverse-tokenizes absolute paths back to $token form on output.
|
|
3
|
+
|
|
4
|
+
Input side (runtime, before processor): $token/subpath -> absolute path
|
|
5
|
+
Output side (context.output / context.meta): absolute path -> $token/subpath
|
|
6
|
+
|
|
7
|
+
Both sides are a no-op when the DPE_* env vars are not set.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
_TOKEN_MAP = [
|
|
14
|
+
("input", "DPE_INPUT"),
|
|
15
|
+
("output", "DPE_OUTPUT"),
|
|
16
|
+
("configs", "DPE_CONFIGS"),
|
|
17
|
+
("storage", "DPE_STORAGE"),
|
|
18
|
+
("temp", "DPE_TEMP"),
|
|
19
|
+
("session", "DPE_SESSION"),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class EnvPaths:
|
|
24
|
+
"""Resolved env prefix map. Build once at tool startup."""
|
|
25
|
+
|
|
26
|
+
__slots__ = ("_entries",)
|
|
27
|
+
|
|
28
|
+
def __init__(self, pairs: list[tuple[str, str]] | None = None):
|
|
29
|
+
"""
|
|
30
|
+
If *pairs* is None the standard DPE_* env vars are read.
|
|
31
|
+
Pass explicit pairs for tests: ``[("input", "/abs/input"), ...]``.
|
|
32
|
+
|
|
33
|
+
Prefer the named factories :meth:`from_env` and :meth:`from_pairs`
|
|
34
|
+
-- they're consistent with the Rust SDK (``from_env()`` /
|
|
35
|
+
``from_pairs(&[...])``) and read more clearly at call sites. The
|
|
36
|
+
polymorphic constructor is kept for backwards compat.
|
|
37
|
+
"""
|
|
38
|
+
if pairs is None:
|
|
39
|
+
raw = []
|
|
40
|
+
for token, env_var in _TOKEN_MAP:
|
|
41
|
+
val = os.environ.get(env_var, "")
|
|
42
|
+
if val:
|
|
43
|
+
raw.append((token, val.replace("\\", "/")))
|
|
44
|
+
else:
|
|
45
|
+
raw = [(t, a.replace("\\", "/")) for t, a in pairs]
|
|
46
|
+
# Sort longest-abs-first for greedy tokenization.
|
|
47
|
+
self._entries: list[tuple[str, str]] = sorted(
|
|
48
|
+
raw, key=lambda e: len(e[1]), reverse=True
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def from_env(cls) -> "EnvPaths":
|
|
53
|
+
"""Read the standard DPE_* env vars; missing/empty vars are skipped."""
|
|
54
|
+
return cls()
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_pairs(cls, pairs: list[tuple[str, str]]) -> "EnvPaths":
|
|
58
|
+
"""Construct from explicit (token, abs_path) pairs -- useful for tests."""
|
|
59
|
+
return cls(pairs)
|
|
60
|
+
|
|
61
|
+
def is_empty(self) -> bool:
|
|
62
|
+
return not self._entries
|
|
63
|
+
|
|
64
|
+
def resolve_value(self, v: Any) -> Any:
|
|
65
|
+
"""Walk v, resolving $token[/subpath] strings to absolute paths."""
|
|
66
|
+
if self.is_empty():
|
|
67
|
+
return v
|
|
68
|
+
return self._walk(v, self._resolve_str)
|
|
69
|
+
|
|
70
|
+
def tokenize_value(self, v: Any) -> Any:
|
|
71
|
+
"""Walk v, tokenizing absolute paths back to $token[/subpath] form."""
|
|
72
|
+
if self.is_empty():
|
|
73
|
+
return v
|
|
74
|
+
return self._walk(v, self._tokenize_str)
|
|
75
|
+
|
|
76
|
+
def _walk(self, v: Any, f) -> Any:
|
|
77
|
+
if isinstance(v, str):
|
|
78
|
+
return f(v)
|
|
79
|
+
if isinstance(v, list):
|
|
80
|
+
return [self._walk(x, f) for x in v]
|
|
81
|
+
if isinstance(v, dict):
|
|
82
|
+
return {k: self._walk(val, f) for k, val in v.items()}
|
|
83
|
+
return v
|
|
84
|
+
|
|
85
|
+
def _resolve_str(self, s: str) -> str:
|
|
86
|
+
if not s.startswith("$"):
|
|
87
|
+
return s
|
|
88
|
+
slash = s.find("/")
|
|
89
|
+
if slash < 0:
|
|
90
|
+
name, tail = s[1:], ""
|
|
91
|
+
else:
|
|
92
|
+
name, tail = s[1:slash], s[slash:]
|
|
93
|
+
for token, abs_path in self._entries:
|
|
94
|
+
if token == name:
|
|
95
|
+
return abs_path + tail if tail else abs_path
|
|
96
|
+
return s
|
|
97
|
+
|
|
98
|
+
def _tokenize_str(self, s: str) -> str:
|
|
99
|
+
normalized = s.replace("\\", "/")
|
|
100
|
+
for token, abs_path in self._entries:
|
|
101
|
+
if normalized.startswith(abs_path):
|
|
102
|
+
rest = normalized[len(abs_path):]
|
|
103
|
+
if not rest:
|
|
104
|
+
return f"${token}"
|
|
105
|
+
if rest.startswith("/"):
|
|
106
|
+
return f"${token}{rest}"
|
|
107
|
+
# rest doesn't start with "/" -- partial component match, skip
|
|
108
|
+
return s
|
|
@@ -10,6 +10,7 @@ from collections.abc import Callable
|
|
|
10
10
|
from dpe._accumulators import Memory
|
|
11
11
|
from dpe._context import Context
|
|
12
12
|
from dpe._envelope import parse_envelope, write_input, write_log
|
|
13
|
+
from dpe._paths import EnvPaths
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class Runtime:
|
|
@@ -22,6 +23,7 @@ class Runtime:
|
|
|
22
23
|
self.processors: dict[str, Callable] = {}
|
|
23
24
|
self._queue: deque = deque()
|
|
24
25
|
self._shutdown = False
|
|
26
|
+
self._paths = EnvPaths()
|
|
25
27
|
|
|
26
28
|
def enqueue(self, queue_name: str, v, id: str, src: str):
|
|
27
29
|
"""Add item to internal queue."""
|
|
@@ -40,7 +42,7 @@ class Runtime:
|
|
|
40
42
|
write_log(f"No processor for queue '{name}', dropping item", "warn")
|
|
41
43
|
continue
|
|
42
44
|
|
|
43
|
-
ctx = Context(item_id, item_src, self.memory, self)
|
|
45
|
+
ctx = Context(item_id, item_src, self.memory, self, self._paths)
|
|
44
46
|
try:
|
|
45
47
|
proc(v, self.settings, ctx)
|
|
46
48
|
except Exception as e:
|
|
@@ -52,7 +54,7 @@ class Runtime:
|
|
|
52
54
|
write_log(f"Queue drain hit safety limit ({max_iterations})", "error")
|
|
53
55
|
|
|
54
56
|
def create_ctx(self, id: str, src: str) -> Context:
|
|
55
|
-
return Context(id, src, self.memory, self)
|
|
57
|
+
return Context(id, src, self.memory, self, self._paths)
|
|
56
58
|
|
|
57
59
|
def _handle_signal(self, signum, frame):
|
|
58
60
|
self._shutdown = True
|
|
@@ -98,6 +100,8 @@ class Runtime:
|
|
|
98
100
|
id = envelope.get("id", "")
|
|
99
101
|
src = envelope.get("src", "")
|
|
100
102
|
v = envelope.get("v", {})
|
|
103
|
+
# Resolve $token paths in v before handing to processor.
|
|
104
|
+
v = self._paths.resolve_value(v)
|
|
101
105
|
|
|
102
106
|
# Emit an `input` event BEFORE the processor runs so the runner
|
|
103
107
|
# can count rows_in for every stage that reads stdin — including
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Integration tests — full tool lifecycle via subprocess."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
import os
|
|
4
5
|
import subprocess
|
|
5
6
|
import sys
|
|
6
7
|
import tempfile
|
|
@@ -546,3 +547,99 @@ class TestContext:
|
|
|
546
547
|
|
|
547
548
|
assert data[0]["v"]["id"] == "my-id"
|
|
548
549
|
assert data[0]["v"]["src"] == "my-src"
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
class TestEnvPathsRoundTrip:
|
|
553
|
+
def _run_with_env(self, code, settings, stdin_lines, env_overrides):
|
|
554
|
+
"""Like run_tool_code but injects custom DPE_* env vars into the subprocess."""
|
|
555
|
+
import subprocess
|
|
556
|
+
import sys
|
|
557
|
+
import tempfile
|
|
558
|
+
import textwrap
|
|
559
|
+
from pathlib import Path
|
|
560
|
+
|
|
561
|
+
stdin_data = (
|
|
562
|
+
"\n".join(json.dumps(r) for r in stdin_lines) + "\n" if stdin_lines else ""
|
|
563
|
+
)
|
|
564
|
+
with tempfile.NamedTemporaryFile(
|
|
565
|
+
mode="w", suffix=".py", delete=False,
|
|
566
|
+
dir=tempfile.gettempdir(), encoding="utf-8",
|
|
567
|
+
) as f:
|
|
568
|
+
f.write(textwrap.dedent(code))
|
|
569
|
+
f.flush()
|
|
570
|
+
tool_path = f.name
|
|
571
|
+
|
|
572
|
+
env = {**os.environ, **env_overrides}
|
|
573
|
+
result = subprocess.run(
|
|
574
|
+
[sys.executable, tool_path, json.dumps(settings)],
|
|
575
|
+
input=stdin_data, capture_output=True, text=True,
|
|
576
|
+
timeout=30, env=env,
|
|
577
|
+
)
|
|
578
|
+
Path(tool_path).unlink(missing_ok=True)
|
|
579
|
+
|
|
580
|
+
data = []
|
|
581
|
+
for line in result.stdout.strip().split("\n"):
|
|
582
|
+
if line:
|
|
583
|
+
rec = json.loads(line)
|
|
584
|
+
if rec.get("t") == "d":
|
|
585
|
+
data.append(rec)
|
|
586
|
+
return data
|
|
587
|
+
|
|
588
|
+
def test_token_resolved_on_input(self):
|
|
589
|
+
"""$input in envelope v is expanded to abs path before processor sees it.
|
|
590
|
+
|
|
591
|
+
The processor inspects the path (not outputs it) so the tokenize-on-output
|
|
592
|
+
step does not reverse the expansion — we test the expansion independently.
|
|
593
|
+
"""
|
|
594
|
+
data = self._run_with_env("""
|
|
595
|
+
import dpe
|
|
596
|
+
|
|
597
|
+
def process_input(v, settings, ctx):
|
|
598
|
+
# Check the resolved value; output a boolean, not the path itself,
|
|
599
|
+
# so the tokenize-on-output step doesn't convert it back.
|
|
600
|
+
ctx.output({"starts_abs": v["path"].startswith("/mnt/drive")})
|
|
601
|
+
|
|
602
|
+
if __name__ == "__main__":
|
|
603
|
+
dpe.run()
|
|
604
|
+
""", {}, [inp({"path": "$input/data.csv"})],
|
|
605
|
+
{"DPE_INPUT": "/mnt/drive"})
|
|
606
|
+
|
|
607
|
+
assert data[0]["v"]["starts_abs"] is True
|
|
608
|
+
|
|
609
|
+
def test_abs_path_tokenized_on_output(self):
|
|
610
|
+
"""Absolute paths in ctx.output(v) are tokenized back to $token form.
|
|
611
|
+
|
|
612
|
+
The processor hardcodes an absolute path string (not from resolved input)
|
|
613
|
+
so we're testing only the output tokenization.
|
|
614
|
+
"""
|
|
615
|
+
data = self._run_with_env("""
|
|
616
|
+
import dpe
|
|
617
|
+
|
|
618
|
+
def process_input(v, settings, ctx):
|
|
619
|
+
# Output a hard-coded abs path -- not from resolved input.
|
|
620
|
+
ctx.output({"path": "/mnt/drive/result.csv"})
|
|
621
|
+
|
|
622
|
+
if __name__ == "__main__":
|
|
623
|
+
dpe.run()
|
|
624
|
+
""", {}, [inp({"x": 1})],
|
|
625
|
+
{"DPE_INPUT": "/mnt/drive"})
|
|
626
|
+
|
|
627
|
+
assert data[0]["v"]["path"] == "$input/result.csv"
|
|
628
|
+
|
|
629
|
+
def test_no_env_passthrough(self):
|
|
630
|
+
"""Without DPE_* vars set, $token paths pass through unchanged."""
|
|
631
|
+
env_overrides = {k: "" for k in [
|
|
632
|
+
"DPE_INPUT", "DPE_OUTPUT", "DPE_CONFIGS",
|
|
633
|
+
"DPE_STORAGE", "DPE_TEMP", "DPE_SESSION",
|
|
634
|
+
]}
|
|
635
|
+
data = self._run_with_env("""
|
|
636
|
+
import dpe
|
|
637
|
+
|
|
638
|
+
def process_input(v, settings, ctx):
|
|
639
|
+
ctx.output(v)
|
|
640
|
+
|
|
641
|
+
if __name__ == "__main__":
|
|
642
|
+
dpe.run()
|
|
643
|
+
""", {}, [inp({"path": "$input/foo"})], env_overrides)
|
|
644
|
+
|
|
645
|
+
assert data[0]["v"]["path"] == "$input/foo"
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Tests for EnvPaths -- $token resolve/tokenize round-trip."""
|
|
2
|
+
|
|
3
|
+
from dpe._paths import EnvPaths
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def paths():
|
|
7
|
+
return EnvPaths([
|
|
8
|
+
("input", "/abs/input"),
|
|
9
|
+
("output", "/abs/output"),
|
|
10
|
+
("storage", "/abs/storage"),
|
|
11
|
+
# Intentionally shorter abs -- tests longest-match logic.
|
|
12
|
+
("session", "/abs/storage/session"),
|
|
13
|
+
])
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# --- resolve_value ---
|
|
17
|
+
|
|
18
|
+
class TestResolveValue:
|
|
19
|
+
def test_top_level_token(self):
|
|
20
|
+
assert paths().resolve_value("$input") == "/abs/input"
|
|
21
|
+
|
|
22
|
+
def test_token_with_subpath(self):
|
|
23
|
+
assert paths().resolve_value("$input/data/file.csv") == "/abs/input/data/file.csv"
|
|
24
|
+
|
|
25
|
+
def test_unknown_token_unchanged(self):
|
|
26
|
+
assert paths().resolve_value("$set/field") == "$set/field"
|
|
27
|
+
assert paths().resolve_value("$bogus") == "$bogus"
|
|
28
|
+
|
|
29
|
+
def test_recurses_into_dict(self):
|
|
30
|
+
v = {"path": "$input/a.csv", "n": 42, "nested": {"p": "$output/b"}}
|
|
31
|
+
r = paths().resolve_value(v)
|
|
32
|
+
assert r["path"] == "/abs/input/a.csv"
|
|
33
|
+
assert r["n"] == 42
|
|
34
|
+
assert r["nested"]["p"] == "/abs/output/b"
|
|
35
|
+
|
|
36
|
+
def test_recurses_into_list(self):
|
|
37
|
+
v = ["$input/x", "$output/y", "plain"]
|
|
38
|
+
r = paths().resolve_value(v)
|
|
39
|
+
assert r == ["/abs/input/x", "/abs/output/y", "plain"]
|
|
40
|
+
|
|
41
|
+
def test_no_env_is_noop(self):
|
|
42
|
+
empty = EnvPaths([])
|
|
43
|
+
v = {"p": "$input/foo"}
|
|
44
|
+
assert empty.resolve_value(v) == v
|
|
45
|
+
|
|
46
|
+
def test_non_string_types_unchanged(self):
|
|
47
|
+
v = {"n": 42, "b": True, "nil": None, "p": "$input"}
|
|
48
|
+
r = paths().resolve_value(v)
|
|
49
|
+
assert r["n"] == 42
|
|
50
|
+
assert r["b"] is True
|
|
51
|
+
assert r["nil"] is None
|
|
52
|
+
assert r["p"] == "/abs/input"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# --- tokenize_value ---
|
|
56
|
+
|
|
57
|
+
class TestTokenizeValue:
|
|
58
|
+
def test_exact_prefix(self):
|
|
59
|
+
assert paths().tokenize_value("/abs/input") == "$input"
|
|
60
|
+
|
|
61
|
+
def test_prefix_with_subpath(self):
|
|
62
|
+
assert paths().tokenize_value("/abs/output/results/out.csv") == "$output/results/out.csv"
|
|
63
|
+
|
|
64
|
+
def test_partial_component_not_replaced(self):
|
|
65
|
+
# "/abs/inputXYZ" must NOT match "$input"
|
|
66
|
+
assert paths().tokenize_value("/abs/inputXYZ") == "/abs/inputXYZ"
|
|
67
|
+
|
|
68
|
+
def test_longest_prefix_wins(self):
|
|
69
|
+
# "/abs/storage/session/..." should match "$session", not "$storage"
|
|
70
|
+
assert paths().tokenize_value("/abs/storage/session/data.json") == "$session/data.json"
|
|
71
|
+
|
|
72
|
+
def test_non_path_unchanged(self):
|
|
73
|
+
assert paths().tokenize_value("hello world") == "hello world"
|
|
74
|
+
assert paths().tokenize_value(42) == 42
|
|
75
|
+
assert paths().tokenize_value(True) is True
|
|
76
|
+
assert paths().tokenize_value(None) is None
|
|
77
|
+
|
|
78
|
+
def test_recurses_into_nested(self):
|
|
79
|
+
v = {"a": {"b": "/abs/input/x"}, "arr": ["/abs/output/y"]}
|
|
80
|
+
r = paths().tokenize_value(v)
|
|
81
|
+
assert r["a"]["b"] == "$input/x"
|
|
82
|
+
assert r["arr"][0] == "$output/y"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# --- round-trip ---
|
|
86
|
+
|
|
87
|
+
class TestRoundTrip:
|
|
88
|
+
def test_resolve_then_tokenize(self):
|
|
89
|
+
p = EnvPaths([("input", "/data/in"), ("output", "/data/out")])
|
|
90
|
+
original = {"src": "$input/file.csv", "dst": "$output/result.csv"}
|
|
91
|
+
resolved = p.resolve_value(original)
|
|
92
|
+
assert resolved == {"src": "/data/in/file.csv", "dst": "/data/out/result.csv"}
|
|
93
|
+
assert p.tokenize_value(resolved) == original
|
|
94
|
+
|
|
95
|
+
def test_windows_backslash_in_abs_normalised(self):
|
|
96
|
+
p = EnvPaths([("data", r"C:\Data\proj")])
|
|
97
|
+
assert p.resolve_value("$data/sub") == "C:/Data/proj/sub"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# --- from env vars ---
|
|
101
|
+
|
|
102
|
+
class TestFromEnv:
|
|
103
|
+
def test_reads_dpe_input(self, monkeypatch):
|
|
104
|
+
monkeypatch.setenv("DPE_INPUT", "/mnt/input")
|
|
105
|
+
monkeypatch.delenv("DPE_OUTPUT", raising=False)
|
|
106
|
+
p = EnvPaths()
|
|
107
|
+
assert p.resolve_value("$input/file.csv") == "/mnt/input/file.csv"
|
|
108
|
+
assert p.resolve_value("$output/x") == "$output/x" # not set
|
|
109
|
+
|
|
110
|
+
def test_missing_vars_empty(self, monkeypatch):
|
|
111
|
+
for v in ["DPE_INPUT", "DPE_OUTPUT", "DPE_CONFIGS",
|
|
112
|
+
"DPE_STORAGE", "DPE_TEMP", "DPE_SESSION"]:
|
|
113
|
+
monkeypatch.delenv(v, raising=False)
|
|
114
|
+
p = EnvPaths()
|
|
115
|
+
assert p.is_empty()
|
|
116
|
+
assert p.resolve_value("$input/foo") == "$input/foo"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# --- named factories (mirror Rust SDK) ---
|
|
120
|
+
|
|
121
|
+
class TestNamedFactories:
|
|
122
|
+
def test_from_env_reads_dpe_vars(self, monkeypatch):
|
|
123
|
+
monkeypatch.setenv("DPE_INPUT", "/mnt/x")
|
|
124
|
+
monkeypatch.delenv("DPE_OUTPUT", raising=False)
|
|
125
|
+
p = EnvPaths.from_env()
|
|
126
|
+
assert p.resolve_value("$input/file") == "/mnt/x/file"
|
|
127
|
+
|
|
128
|
+
def test_from_pairs_bypasses_env(self, monkeypatch):
|
|
129
|
+
monkeypatch.setenv("DPE_INPUT", "/ignored")
|
|
130
|
+
p = EnvPaths.from_pairs([("input", "/explicit")])
|
|
131
|
+
assert p.resolve_value("$input/file") == "/explicit/file"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|