combycode-dpe 2.0.1__py3-none-any.whl → 2.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {combycode_dpe-2.0.1.dist-info → combycode_dpe-2.0.2.dist-info}/METADATA +1 -1
- combycode_dpe-2.0.2.dist-info/RECORD +10 -0
- dpe/__init__.py +2 -0
- dpe/_context.py +11 -7
- dpe/_paths.py +108 -0
- dpe/_runtime.py +6 -2
- combycode_dpe-2.0.1.dist-info/RECORD +0 -9
- {combycode_dpe-2.0.1.dist-info → combycode_dpe-2.0.2.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
dpe/__init__.py,sha256=Ep_fj9472z6EyIZFE5u8e7jjONMowUVfAfCPxySw3Tw,1550
|
|
2
|
+
dpe/_accumulators.py,sha256=7m2rP4qQ_Kh9AGU-iwC53jcS08GVQE4wqGMGa5Yj6pc,7072
|
|
3
|
+
dpe/_cache.py,sha256=Nd6UfNhRxA3ZivYewrRQ_-CTG1NI_Y2F3KBCX7Inyk8,2809
|
|
4
|
+
dpe/_context.py,sha256=Nqe5hJCk1T1WFPFwAhCAvIxVKDt9E_vHceHh8KcwdqI,5638
|
|
5
|
+
dpe/_envelope.py,sha256=H9oqSUqUcR-ipKbyTO81cEg0KNLzi8ND-aKwDn8N7So,4455
|
|
6
|
+
dpe/_paths.py,sha256=AiGtR_ULKHLgBL9ali4c9wIfo7e1FcKnwiTRNmxqMPU,3728
|
|
7
|
+
dpe/_runtime.py,sha256=RcrJzp4U9koL7ey2YiO7-xCk0bm2QQU2omGur3PfnXQ,5539
|
|
8
|
+
combycode_dpe-2.0.2.dist-info/METADATA,sha256=3ptTAa7AXnqBq2m_mEoizk6A4pbwAV7rLSZ00Ve-XqQ,418
|
|
9
|
+
combycode_dpe-2.0.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
10
|
+
combycode_dpe-2.0.2.dist-info/RECORD,,
|
dpe/__init__.py
CHANGED
|
@@ -36,6 +36,7 @@ from dpe._accumulators import (
|
|
|
36
36
|
)
|
|
37
37
|
from dpe._context import Context
|
|
38
38
|
from dpe._envelope import hash_file, hash_string
|
|
39
|
+
from dpe._paths import EnvPaths
|
|
39
40
|
from dpe._runtime import Runtime
|
|
40
41
|
|
|
41
42
|
|
|
@@ -63,6 +64,7 @@ __all__ = [
|
|
|
63
64
|
"hash_string",
|
|
64
65
|
"hash_file",
|
|
65
66
|
"Context",
|
|
67
|
+
"EnvPaths",
|
|
66
68
|
"Memory",
|
|
67
69
|
"Number",
|
|
68
70
|
"Average",
|
dpe/_context.py
CHANGED
|
@@ -15,6 +15,7 @@ from dpe._envelope import (
|
|
|
15
15
|
write_stats,
|
|
16
16
|
write_trace,
|
|
17
17
|
)
|
|
18
|
+
from dpe._paths import EnvPaths
|
|
18
19
|
|
|
19
20
|
T = TypeVar("T")
|
|
20
21
|
|
|
@@ -27,13 +28,15 @@ class Context:
|
|
|
27
28
|
Memory is shared across all processors in the tool.
|
|
28
29
|
"""
|
|
29
30
|
|
|
30
|
-
__slots__ = ("id", "src", "memory", "_runtime", "_labels")
|
|
31
|
+
__slots__ = ("id", "src", "memory", "_runtime", "_paths", "_labels")
|
|
31
32
|
|
|
32
|
-
def __init__(self, id: str, src: str, memory: Memory, runtime
|
|
33
|
+
def __init__(self, id: str, src: str, memory: Memory, runtime,
|
|
34
|
+
paths: EnvPaths | None = None):
|
|
33
35
|
self.id = id
|
|
34
36
|
self.src = src
|
|
35
37
|
self.memory = memory
|
|
36
38
|
self._runtime = runtime
|
|
39
|
+
self._paths = paths if paths is not None else EnvPaths()
|
|
37
40
|
# Labels accumulated by ctx.trace(k, v). Flushed as one merged
|
|
38
41
|
# {type:"trace"} stderr event before each ctx.output(), then cleared.
|
|
39
42
|
self._labels: dict = {}
|
|
@@ -57,15 +60,16 @@ class Context:
|
|
|
57
60
|
"""Emit data record to stdout.
|
|
58
61
|
|
|
59
62
|
Flushes accumulated trace labels as a merged trace event first
|
|
60
|
-
(channel="data"
|
|
63
|
+
(channel="data" -> counts as rows_out at the runner), then writes
|
|
61
64
|
the envelope, then clears the label bag.
|
|
65
|
+
Absolute paths in v are reverse-tokenized to $token/... form.
|
|
62
66
|
"""
|
|
63
67
|
out_id = id if id is not None else self.id
|
|
64
68
|
out_src = src if src is not None else self.src
|
|
65
|
-
# Emit trace even with empty labels
|
|
69
|
+
# Emit trace even with empty labels -- the chain row itself is the value.
|
|
66
70
|
write_trace(out_id, out_src, self._labels, channel="data")
|
|
67
71
|
self._labels = {}
|
|
68
|
-
write_data(v, out_id, out_src)
|
|
72
|
+
write_data(self._paths.tokenize_value(v), out_id, out_src)
|
|
69
73
|
|
|
70
74
|
def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
|
|
71
75
|
"""Emit to internal named queue. Processed by process_<queue>().
|
|
@@ -93,10 +97,10 @@ class Context:
|
|
|
93
97
|
|
|
94
98
|
Also emits a {type:"trace", channel:"meta"} stderr event so the
|
|
95
99
|
runner can increment its per-stage `meta` counter. Inherits
|
|
96
|
-
ctx id/src.
|
|
100
|
+
ctx id/src. Absolute paths in v are reverse-tokenized to $token/... form.
|
|
97
101
|
"""
|
|
98
102
|
write_trace(self.id, self.src, {}, channel="meta")
|
|
99
|
-
write_meta(v)
|
|
103
|
+
write_meta(self._paths.tokenize_value(v))
|
|
100
104
|
|
|
101
105
|
def log(self, msg: str, *, level: str = "info", **extra):
|
|
102
106
|
"""Write structured log to stderr."""
|
dpe/_paths.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""EnvPaths — resolves $token path prefixes in envelope v on input;
|
|
2
|
+
reverse-tokenizes absolute paths back to $token form on output.
|
|
3
|
+
|
|
4
|
+
Input side (runtime, before processor): $token/subpath -> absolute path
|
|
5
|
+
Output side (context.output / context.meta): absolute path -> $token/subpath
|
|
6
|
+
|
|
7
|
+
Both sides are a no-op when the DPE_* env vars are not set.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
_TOKEN_MAP = [
|
|
14
|
+
("input", "DPE_INPUT"),
|
|
15
|
+
("output", "DPE_OUTPUT"),
|
|
16
|
+
("configs", "DPE_CONFIGS"),
|
|
17
|
+
("storage", "DPE_STORAGE"),
|
|
18
|
+
("temp", "DPE_TEMP"),
|
|
19
|
+
("session", "DPE_SESSION"),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class EnvPaths:
|
|
24
|
+
"""Resolved env prefix map. Build once at tool startup."""
|
|
25
|
+
|
|
26
|
+
__slots__ = ("_entries",)
|
|
27
|
+
|
|
28
|
+
def __init__(self, pairs: list[tuple[str, str]] | None = None):
|
|
29
|
+
"""
|
|
30
|
+
If *pairs* is None the standard DPE_* env vars are read.
|
|
31
|
+
Pass explicit pairs for tests: ``[("input", "/abs/input"), ...]``.
|
|
32
|
+
|
|
33
|
+
Prefer the named factories :meth:`from_env` and :meth:`from_pairs`
|
|
34
|
+
-- they're consistent with the Rust SDK (``from_env()`` /
|
|
35
|
+
``from_pairs(&[...])``) and read more clearly at call sites. The
|
|
36
|
+
polymorphic constructor is kept for backwards compat.
|
|
37
|
+
"""
|
|
38
|
+
if pairs is None:
|
|
39
|
+
raw = []
|
|
40
|
+
for token, env_var in _TOKEN_MAP:
|
|
41
|
+
val = os.environ.get(env_var, "")
|
|
42
|
+
if val:
|
|
43
|
+
raw.append((token, val.replace("\\", "/")))
|
|
44
|
+
else:
|
|
45
|
+
raw = [(t, a.replace("\\", "/")) for t, a in pairs]
|
|
46
|
+
# Sort longest-abs-first for greedy tokenization.
|
|
47
|
+
self._entries: list[tuple[str, str]] = sorted(
|
|
48
|
+
raw, key=lambda e: len(e[1]), reverse=True
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def from_env(cls) -> "EnvPaths":
|
|
53
|
+
"""Read the standard DPE_* env vars; missing/empty vars are skipped."""
|
|
54
|
+
return cls()
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_pairs(cls, pairs: list[tuple[str, str]]) -> "EnvPaths":
|
|
58
|
+
"""Construct from explicit (token, abs_path) pairs -- useful for tests."""
|
|
59
|
+
return cls(pairs)
|
|
60
|
+
|
|
61
|
+
def is_empty(self) -> bool:
|
|
62
|
+
return not self._entries
|
|
63
|
+
|
|
64
|
+
def resolve_value(self, v: Any) -> Any:
|
|
65
|
+
"""Walk v, resolving $token[/subpath] strings to absolute paths."""
|
|
66
|
+
if self.is_empty():
|
|
67
|
+
return v
|
|
68
|
+
return self._walk(v, self._resolve_str)
|
|
69
|
+
|
|
70
|
+
def tokenize_value(self, v: Any) -> Any:
|
|
71
|
+
"""Walk v, tokenizing absolute paths back to $token[/subpath] form."""
|
|
72
|
+
if self.is_empty():
|
|
73
|
+
return v
|
|
74
|
+
return self._walk(v, self._tokenize_str)
|
|
75
|
+
|
|
76
|
+
def _walk(self, v: Any, f) -> Any:
|
|
77
|
+
if isinstance(v, str):
|
|
78
|
+
return f(v)
|
|
79
|
+
if isinstance(v, list):
|
|
80
|
+
return [self._walk(x, f) for x in v]
|
|
81
|
+
if isinstance(v, dict):
|
|
82
|
+
return {k: self._walk(val, f) for k, val in v.items()}
|
|
83
|
+
return v
|
|
84
|
+
|
|
85
|
+
def _resolve_str(self, s: str) -> str:
|
|
86
|
+
if not s.startswith("$"):
|
|
87
|
+
return s
|
|
88
|
+
slash = s.find("/")
|
|
89
|
+
if slash < 0:
|
|
90
|
+
name, tail = s[1:], ""
|
|
91
|
+
else:
|
|
92
|
+
name, tail = s[1:slash], s[slash:]
|
|
93
|
+
for token, abs_path in self._entries:
|
|
94
|
+
if token == name:
|
|
95
|
+
return abs_path + tail if tail else abs_path
|
|
96
|
+
return s
|
|
97
|
+
|
|
98
|
+
def _tokenize_str(self, s: str) -> str:
|
|
99
|
+
normalized = s.replace("\\", "/")
|
|
100
|
+
for token, abs_path in self._entries:
|
|
101
|
+
if normalized.startswith(abs_path):
|
|
102
|
+
rest = normalized[len(abs_path):]
|
|
103
|
+
if not rest:
|
|
104
|
+
return f"${token}"
|
|
105
|
+
if rest.startswith("/"):
|
|
106
|
+
return f"${token}{rest}"
|
|
107
|
+
# rest doesn't start with "/" -- partial component match, skip
|
|
108
|
+
return s
|
dpe/_runtime.py
CHANGED
|
@@ -10,6 +10,7 @@ from collections.abc import Callable
|
|
|
10
10
|
from dpe._accumulators import Memory
|
|
11
11
|
from dpe._context import Context
|
|
12
12
|
from dpe._envelope import parse_envelope, write_input, write_log
|
|
13
|
+
from dpe._paths import EnvPaths
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class Runtime:
|
|
@@ -22,6 +23,7 @@ class Runtime:
|
|
|
22
23
|
self.processors: dict[str, Callable] = {}
|
|
23
24
|
self._queue: deque = deque()
|
|
24
25
|
self._shutdown = False
|
|
26
|
+
self._paths = EnvPaths()
|
|
25
27
|
|
|
26
28
|
def enqueue(self, queue_name: str, v, id: str, src: str):
|
|
27
29
|
"""Add item to internal queue."""
|
|
@@ -40,7 +42,7 @@ class Runtime:
|
|
|
40
42
|
write_log(f"No processor for queue '{name}', dropping item", "warn")
|
|
41
43
|
continue
|
|
42
44
|
|
|
43
|
-
ctx = Context(item_id, item_src, self.memory, self)
|
|
45
|
+
ctx = Context(item_id, item_src, self.memory, self, self._paths)
|
|
44
46
|
try:
|
|
45
47
|
proc(v, self.settings, ctx)
|
|
46
48
|
except Exception as e:
|
|
@@ -52,7 +54,7 @@ class Runtime:
|
|
|
52
54
|
write_log(f"Queue drain hit safety limit ({max_iterations})", "error")
|
|
53
55
|
|
|
54
56
|
def create_ctx(self, id: str, src: str) -> Context:
|
|
55
|
-
return Context(id, src, self.memory, self)
|
|
57
|
+
return Context(id, src, self.memory, self, self._paths)
|
|
56
58
|
|
|
57
59
|
def _handle_signal(self, signum, frame):
|
|
58
60
|
self._shutdown = True
|
|
@@ -98,6 +100,8 @@ class Runtime:
|
|
|
98
100
|
id = envelope.get("id", "")
|
|
99
101
|
src = envelope.get("src", "")
|
|
100
102
|
v = envelope.get("v", {})
|
|
103
|
+
# Resolve $token paths in v before handing to processor.
|
|
104
|
+
v = self._paths.resolve_value(v)
|
|
101
105
|
|
|
102
106
|
# Emit an `input` event BEFORE the processor runs so the runner
|
|
103
107
|
# can count rows_in for every stage that reads stdin — including
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
dpe/__init__.py,sha256=wS3jsykh8cjhoPleB67N2S56hxCLGV97NH4-OZTJBK4,1502
|
|
2
|
-
dpe/_accumulators.py,sha256=7m2rP4qQ_Kh9AGU-iwC53jcS08GVQE4wqGMGa5Yj6pc,7072
|
|
3
|
-
dpe/_cache.py,sha256=Nd6UfNhRxA3ZivYewrRQ_-CTG1NI_Y2F3KBCX7Inyk8,2809
|
|
4
|
-
dpe/_context.py,sha256=sp-Anum1_6X2_RRFVDmphYnNNOZ6-_VE4EcT5hMIlsk,5297
|
|
5
|
-
dpe/_envelope.py,sha256=H9oqSUqUcR-ipKbyTO81cEg0KNLzi8ND-aKwDn8N7So,4455
|
|
6
|
-
dpe/_runtime.py,sha256=XMtnyN23a0NsAox537c26uUJJWDIB8NwhJz_3R2CPTo,5334
|
|
7
|
-
combycode_dpe-2.0.1.dist-info/METADATA,sha256=v0_4YheOwA0viZN52uWvvr1xwqQRWPmTncVGfzMMat0,418
|
|
8
|
-
combycode_dpe-2.0.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
9
|
-
combycode_dpe-2.0.1.dist-info/RECORD,,
|
|
File without changes
|