combycode-dpe 2.0.1__py3-none-any.whl → 2.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: combycode-dpe
3
- Version: 2.0.1
3
+ Version: 2.0.2
4
4
  Summary: DPE framework for building streaming pipeline tools in Python
5
5
  Project-URL: Repository, https://github.com/combycode/dpe-framework-python
6
6
  License-Expression: AGPL-3.0-or-later
@@ -0,0 +1,10 @@
1
+ dpe/__init__.py,sha256=Ep_fj9472z6EyIZFE5u8e7jjONMowUVfAfCPxySw3Tw,1550
2
+ dpe/_accumulators.py,sha256=7m2rP4qQ_Kh9AGU-iwC53jcS08GVQE4wqGMGa5Yj6pc,7072
3
+ dpe/_cache.py,sha256=Nd6UfNhRxA3ZivYewrRQ_-CTG1NI_Y2F3KBCX7Inyk8,2809
4
+ dpe/_context.py,sha256=Nqe5hJCk1T1WFPFwAhCAvIxVKDt9E_vHceHh8KcwdqI,5638
5
+ dpe/_envelope.py,sha256=H9oqSUqUcR-ipKbyTO81cEg0KNLzi8ND-aKwDn8N7So,4455
6
+ dpe/_paths.py,sha256=AiGtR_ULKHLgBL9ali4c9wIfo7e1FcKnwiTRNmxqMPU,3728
7
+ dpe/_runtime.py,sha256=RcrJzp4U9koL7ey2YiO7-xCk0bm2QQU2omGur3PfnXQ,5539
8
+ combycode_dpe-2.0.2.dist-info/METADATA,sha256=3ptTAa7AXnqBq2m_mEoizk6A4pbwAV7rLSZ00Ve-XqQ,418
9
+ combycode_dpe-2.0.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
10
+ combycode_dpe-2.0.2.dist-info/RECORD,,
dpe/__init__.py CHANGED
@@ -36,6 +36,7 @@ from dpe._accumulators import (
36
36
  )
37
37
  from dpe._context import Context
38
38
  from dpe._envelope import hash_file, hash_string
39
+ from dpe._paths import EnvPaths
39
40
  from dpe._runtime import Runtime
40
41
 
41
42
 
@@ -63,6 +64,7 @@ __all__ = [
63
64
  "hash_string",
64
65
  "hash_file",
65
66
  "Context",
67
+ "EnvPaths",
66
68
  "Memory",
67
69
  "Number",
68
70
  "Average",
dpe/_context.py CHANGED
@@ -15,6 +15,7 @@ from dpe._envelope import (
15
15
  write_stats,
16
16
  write_trace,
17
17
  )
18
+ from dpe._paths import EnvPaths
18
19
 
19
20
  T = TypeVar("T")
20
21
 
@@ -27,13 +28,15 @@ class Context:
27
28
  Memory is shared across all processors in the tool.
28
29
  """
29
30
 
30
- __slots__ = ("id", "src", "memory", "_runtime", "_labels")
31
+ __slots__ = ("id", "src", "memory", "_runtime", "_paths", "_labels")
31
32
 
32
- def __init__(self, id: str, src: str, memory: Memory, runtime):
33
+ def __init__(self, id: str, src: str, memory: Memory, runtime,
34
+ paths: EnvPaths | None = None):
33
35
  self.id = id
34
36
  self.src = src
35
37
  self.memory = memory
36
38
  self._runtime = runtime
39
+ self._paths = paths if paths is not None else EnvPaths()
37
40
  # Labels accumulated by ctx.trace(k, v). Flushed as one merged
38
41
  # {type:"trace"} stderr event before each ctx.output(), then cleared.
39
42
  self._labels: dict = {}
@@ -57,15 +60,16 @@ class Context:
57
60
  """Emit data record to stdout.
58
61
 
59
62
  Flushes accumulated trace labels as a merged trace event first
60
- (channel="data" → counts as rows_out at the runner), then writes
63
+ (channel="data" -> counts as rows_out at the runner), then writes
61
64
  the envelope, then clears the label bag.
65
+ Absolute paths in v are reverse-tokenized to $token/... form.
62
66
  """
63
67
  out_id = id if id is not None else self.id
64
68
  out_src = src if src is not None else self.src
65
- # Emit trace even with empty labels — the chain row itself is the value.
69
+ # Emit trace even with empty labels -- the chain row itself is the value.
66
70
  write_trace(out_id, out_src, self._labels, channel="data")
67
71
  self._labels = {}
68
- write_data(v, out_id, out_src)
72
+ write_data(self._paths.tokenize_value(v), out_id, out_src)
69
73
 
70
74
  def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
71
75
  """Emit to internal named queue. Processed by process_<queue>().
@@ -93,10 +97,10 @@ class Context:
93
97
 
94
98
  Also emits a {type:"trace", channel:"meta"} stderr event so the
95
99
  runner can increment its per-stage `meta` counter. Inherits
96
- ctx id/src.
100
+ ctx id/src. Absolute paths in v are reverse-tokenized to $token/... form.
97
101
  """
98
102
  write_trace(self.id, self.src, {}, channel="meta")
99
- write_meta(v)
103
+ write_meta(self._paths.tokenize_value(v))
100
104
 
101
105
  def log(self, msg: str, *, level: str = "info", **extra):
102
106
  """Write structured log to stderr."""
dpe/_paths.py ADDED
@@ -0,0 +1,108 @@
1
+ """EnvPaths — resolves $token path prefixes in envelope v on input;
2
+ reverse-tokenizes absolute paths back to $token form on output.
3
+
4
+ Input side (runtime, before processor): $token/subpath -> absolute path
5
+ Output side (context.output / context.meta): absolute path -> $token/subpath
6
+
7
+ Both sides are a no-op when the DPE_* env vars are not set.
8
+ """
9
+
10
+ import os
11
+ from typing import Any
12
+
13
+ _TOKEN_MAP = [
14
+ ("input", "DPE_INPUT"),
15
+ ("output", "DPE_OUTPUT"),
16
+ ("configs", "DPE_CONFIGS"),
17
+ ("storage", "DPE_STORAGE"),
18
+ ("temp", "DPE_TEMP"),
19
+ ("session", "DPE_SESSION"),
20
+ ]
21
+
22
+
23
+ class EnvPaths:
24
+ """Resolved env prefix map. Build once at tool startup."""
25
+
26
+ __slots__ = ("_entries",)
27
+
28
+ def __init__(self, pairs: list[tuple[str, str]] | None = None):
29
+ """
30
+ If *pairs* is None the standard DPE_* env vars are read.
31
+ Pass explicit pairs for tests: ``[("input", "/abs/input"), ...]``.
32
+
33
+ Prefer the named factories :meth:`from_env` and :meth:`from_pairs`
34
+ -- they're consistent with the Rust SDK (``from_env()`` /
35
+ ``from_pairs(&[...])``) and read more clearly at call sites. The
36
+ polymorphic constructor is kept for backwards compat.
37
+ """
38
+ if pairs is None:
39
+ raw = []
40
+ for token, env_var in _TOKEN_MAP:
41
+ val = os.environ.get(env_var, "")
42
+ if val:
43
+ raw.append((token, val.replace("\\", "/")))
44
+ else:
45
+ raw = [(t, a.replace("\\", "/")) for t, a in pairs]
46
+ # Sort longest-abs-first for greedy tokenization.
47
+ self._entries: list[tuple[str, str]] = sorted(
48
+ raw, key=lambda e: len(e[1]), reverse=True
49
+ )
50
+
51
+ @classmethod
52
+ def from_env(cls) -> "EnvPaths":
53
+ """Read the standard DPE_* env vars; missing/empty vars are skipped."""
54
+ return cls()
55
+
56
+ @classmethod
57
+ def from_pairs(cls, pairs: list[tuple[str, str]]) -> "EnvPaths":
58
+ """Construct from explicit (token, abs_path) pairs -- useful for tests."""
59
+ return cls(pairs)
60
+
61
+ def is_empty(self) -> bool:
62
+ return not self._entries
63
+
64
+ def resolve_value(self, v: Any) -> Any:
65
+ """Walk v, resolving $token[/subpath] strings to absolute paths."""
66
+ if self.is_empty():
67
+ return v
68
+ return self._walk(v, self._resolve_str)
69
+
70
+ def tokenize_value(self, v: Any) -> Any:
71
+ """Walk v, tokenizing absolute paths back to $token[/subpath] form."""
72
+ if self.is_empty():
73
+ return v
74
+ return self._walk(v, self._tokenize_str)
75
+
76
+ def _walk(self, v: Any, f) -> Any:
77
+ if isinstance(v, str):
78
+ return f(v)
79
+ if isinstance(v, list):
80
+ return [self._walk(x, f) for x in v]
81
+ if isinstance(v, dict):
82
+ return {k: self._walk(val, f) for k, val in v.items()}
83
+ return v
84
+
85
+ def _resolve_str(self, s: str) -> str:
86
+ if not s.startswith("$"):
87
+ return s
88
+ slash = s.find("/")
89
+ if slash < 0:
90
+ name, tail = s[1:], ""
91
+ else:
92
+ name, tail = s[1:slash], s[slash:]
93
+ for token, abs_path in self._entries:
94
+ if token == name:
95
+ return abs_path + tail if tail else abs_path
96
+ return s
97
+
98
+ def _tokenize_str(self, s: str) -> str:
99
+ normalized = s.replace("\\", "/")
100
+ for token, abs_path in self._entries:
101
+ if normalized.startswith(abs_path):
102
+ rest = normalized[len(abs_path):]
103
+ if not rest:
104
+ return f"${token}"
105
+ if rest.startswith("/"):
106
+ return f"${token}{rest}"
107
+ # rest doesn't start with "/" -- partial component match, skip
108
+ return s
dpe/_runtime.py CHANGED
@@ -10,6 +10,7 @@ from collections.abc import Callable
10
10
  from dpe._accumulators import Memory
11
11
  from dpe._context import Context
12
12
  from dpe._envelope import parse_envelope, write_input, write_log
13
+ from dpe._paths import EnvPaths
13
14
 
14
15
 
15
16
  class Runtime:
@@ -22,6 +23,7 @@ class Runtime:
22
23
  self.processors: dict[str, Callable] = {}
23
24
  self._queue: deque = deque()
24
25
  self._shutdown = False
26
+ self._paths = EnvPaths()
25
27
 
26
28
  def enqueue(self, queue_name: str, v, id: str, src: str):
27
29
  """Add item to internal queue."""
@@ -40,7 +42,7 @@ class Runtime:
40
42
  write_log(f"No processor for queue '{name}', dropping item", "warn")
41
43
  continue
42
44
 
43
- ctx = Context(item_id, item_src, self.memory, self)
45
+ ctx = Context(item_id, item_src, self.memory, self, self._paths)
44
46
  try:
45
47
  proc(v, self.settings, ctx)
46
48
  except Exception as e:
@@ -52,7 +54,7 @@ class Runtime:
52
54
  write_log(f"Queue drain hit safety limit ({max_iterations})", "error")
53
55
 
54
56
  def create_ctx(self, id: str, src: str) -> Context:
55
- return Context(id, src, self.memory, self)
57
+ return Context(id, src, self.memory, self, self._paths)
56
58
 
57
59
  def _handle_signal(self, signum, frame):
58
60
  self._shutdown = True
@@ -98,6 +100,8 @@ class Runtime:
98
100
  id = envelope.get("id", "")
99
101
  src = envelope.get("src", "")
100
102
  v = envelope.get("v", {})
103
+ # Resolve $token paths in v before handing to processor.
104
+ v = self._paths.resolve_value(v)
101
105
 
102
106
  # Emit an `input` event BEFORE the processor runs so the runner
103
107
  # can count rows_in for every stage that reads stdin — including
@@ -1,9 +0,0 @@
1
- dpe/__init__.py,sha256=wS3jsykh8cjhoPleB67N2S56hxCLGV97NH4-OZTJBK4,1502
2
- dpe/_accumulators.py,sha256=7m2rP4qQ_Kh9AGU-iwC53jcS08GVQE4wqGMGa5Yj6pc,7072
3
- dpe/_cache.py,sha256=Nd6UfNhRxA3ZivYewrRQ_-CTG1NI_Y2F3KBCX7Inyk8,2809
4
- dpe/_context.py,sha256=sp-Anum1_6X2_RRFVDmphYnNNOZ6-_VE4EcT5hMIlsk,5297
5
- dpe/_envelope.py,sha256=H9oqSUqUcR-ipKbyTO81cEg0KNLzi8ND-aKwDn8N7So,4455
6
- dpe/_runtime.py,sha256=XMtnyN23a0NsAox537c26uUJJWDIB8NwhJz_3R2CPTo,5334
7
- combycode_dpe-2.0.1.dist-info/METADATA,sha256=v0_4YheOwA0viZN52uWvvr1xwqQRWPmTncVGfzMMat0,418
8
- combycode_dpe-2.0.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
9
- combycode_dpe-2.0.1.dist-info/RECORD,,