combycode-dpe 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- combycode_dpe-2.0.0.dist-info/METADATA +11 -0
- combycode_dpe-2.0.0.dist-info/RECORD +8 -0
- combycode_dpe-2.0.0.dist-info/WHEEL +4 -0
- dpe/__init__.py +76 -0
- dpe/_accumulators.py +290 -0
- dpe/_context.py +102 -0
- dpe/_envelope.py +116 -0
- dpe/_runtime.py +136 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: combycode-dpe
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: DPE framework for building streaming pipeline tools in Python
|
|
5
|
+
Project-URL: Repository, https://github.com/combycode/dpe-framework-python
|
|
6
|
+
License-Expression: AGPL-3.0-or-later
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
Requires-Dist: jsonschema>=4.20; extra == 'dev'
|
|
10
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
11
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
dpe/__init__.py,sha256=wS3jsykh8cjhoPleB67N2S56hxCLGV97NH4-OZTJBK4,1502
|
|
2
|
+
dpe/_accumulators.py,sha256=7m2rP4qQ_Kh9AGU-iwC53jcS08GVQE4wqGMGa5Yj6pc,7072
|
|
3
|
+
dpe/_context.py,sha256=dOCb5GQQK65f37_mra11eP73YhgS483j_Ly715nUFjI,3648
|
|
4
|
+
dpe/_envelope.py,sha256=gp2VD2dH_Or8vhU287PlQhi1wrn_H4S0NG5wZLs23n8,3756
|
|
5
|
+
dpe/_runtime.py,sha256=YNfms5VL-Ztm9OygkoNp8mCqWK4_LCy6_qbsNZWYssI,4331
|
|
6
|
+
combycode_dpe-2.0.0.dist-info/METADATA,sha256=DJzw9AEzPddVFJ1iLvq0BHJIEXwi48uZ0BqIMeum-lo,418
|
|
7
|
+
combycode_dpe-2.0.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
8
|
+
combycode_dpe-2.0.0.dist-info/RECORD,,
|
dpe/__init__.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DPE — Data Processing Engine framework.
|
|
3
|
+
|
|
4
|
+
Build streaming pipeline tools with minimal boilerplate.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
import dpe
|
|
8
|
+
|
|
9
|
+
def process_input(v, settings, ctx):
|
|
10
|
+
ctx.output({"result": transform(v)})
|
|
11
|
+
ctx.emit("validate", {"data": v})
|
|
12
|
+
|
|
13
|
+
def process_validate(v, settings, ctx):
|
|
14
|
+
if ok(v):
|
|
15
|
+
ctx.output(v)
|
|
16
|
+
else:
|
|
17
|
+
ctx.error(v, "validation failed")
|
|
18
|
+
|
|
19
|
+
if __name__ == "__main__":
|
|
20
|
+
dpe.run()
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import inspect
|
|
24
|
+
|
|
25
|
+
from dpe._accumulators import (
|
|
26
|
+
Average,
|
|
27
|
+
BitMask,
|
|
28
|
+
Boolean,
|
|
29
|
+
Buffer,
|
|
30
|
+
Map,
|
|
31
|
+
Memory,
|
|
32
|
+
MinMax,
|
|
33
|
+
Number,
|
|
34
|
+
Set,
|
|
35
|
+
Trigger,
|
|
36
|
+
)
|
|
37
|
+
from dpe._context import Context
|
|
38
|
+
from dpe._envelope import hash_file, hash_string
|
|
39
|
+
from dpe._runtime import Runtime
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def run():
|
|
43
|
+
"""Start the tool processing loop.
|
|
44
|
+
|
|
45
|
+
Discovers process_input and process_<queue> functions in the calling module,
|
|
46
|
+
parses settings from argv[1], reads stdin line by line, and dispatches.
|
|
47
|
+
|
|
48
|
+
Call this at the bottom of your tool:
|
|
49
|
+
if __name__ == "__main__":
|
|
50
|
+
dpe.run()
|
|
51
|
+
"""
|
|
52
|
+
frame = inspect.stack()[1]
|
|
53
|
+
caller_module = inspect.getmodule(frame[0])
|
|
54
|
+
if caller_module is None:
|
|
55
|
+
raise RuntimeError("Cannot discover caller module")
|
|
56
|
+
|
|
57
|
+
runtime = Runtime()
|
|
58
|
+
runtime.run(caller_module)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
__all__ = [
|
|
62
|
+
"run",
|
|
63
|
+
"hash_string",
|
|
64
|
+
"hash_file",
|
|
65
|
+
"Context",
|
|
66
|
+
"Memory",
|
|
67
|
+
"Number",
|
|
68
|
+
"Average",
|
|
69
|
+
"MinMax",
|
|
70
|
+
"Set",
|
|
71
|
+
"Map",
|
|
72
|
+
"Buffer",
|
|
73
|
+
"Boolean",
|
|
74
|
+
"BitMask",
|
|
75
|
+
"Trigger",
|
|
76
|
+
]
|
dpe/_accumulators.py
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Typed accumulators for shared memory across processors."""
|
|
2
|
+
|
|
3
|
+
from collections import deque
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Number:
|
|
7
|
+
"""Numeric accumulator with increment/decrement."""
|
|
8
|
+
|
|
9
|
+
__slots__ = ("_value",)
|
|
10
|
+
|
|
11
|
+
def __init__(self, initial: float = 0):
|
|
12
|
+
self._value = initial
|
|
13
|
+
|
|
14
|
+
def inc(self, delta: float = 1) -> float:
|
|
15
|
+
self._value += delta
|
|
16
|
+
return self._value
|
|
17
|
+
|
|
18
|
+
def dec(self, delta: float = 1) -> float:
|
|
19
|
+
self._value -= delta
|
|
20
|
+
return self._value
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def value(self) -> float:
|
|
24
|
+
return self._value
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Average:
|
|
28
|
+
"""Running average without storing all values."""
|
|
29
|
+
|
|
30
|
+
__slots__ = ("_sum", "_count")
|
|
31
|
+
|
|
32
|
+
def __init__(self):
|
|
33
|
+
self._sum = 0.0
|
|
34
|
+
self._count = 0
|
|
35
|
+
|
|
36
|
+
def add(self, value: float):
|
|
37
|
+
self._sum += value
|
|
38
|
+
self._count += 1
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def value(self) -> float:
|
|
42
|
+
if self._count == 0:
|
|
43
|
+
return 0.0
|
|
44
|
+
return self._sum / self._count
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def count(self) -> int:
|
|
48
|
+
return self._count
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class MinMax:
|
|
52
|
+
"""Track minimum and maximum values."""
|
|
53
|
+
|
|
54
|
+
__slots__ = ("_min", "_max", "_count")
|
|
55
|
+
|
|
56
|
+
def __init__(self):
|
|
57
|
+
self._min = None
|
|
58
|
+
self._max = None
|
|
59
|
+
self._count = 0
|
|
60
|
+
|
|
61
|
+
def add(self, value: float):
|
|
62
|
+
if self._min is None or value < self._min:
|
|
63
|
+
self._min = value
|
|
64
|
+
if self._max is None or value > self._max:
|
|
65
|
+
self._max = value
|
|
66
|
+
self._count += 1
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def min(self) -> float | None:
|
|
70
|
+
return self._min
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def max(self) -> float | None:
|
|
74
|
+
return self._max
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def count(self) -> int:
|
|
78
|
+
return self._count
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Set:
|
|
82
|
+
"""Unique value collection."""
|
|
83
|
+
|
|
84
|
+
__slots__ = ("_data",)
|
|
85
|
+
|
|
86
|
+
def __init__(self):
|
|
87
|
+
self._data = set()
|
|
88
|
+
|
|
89
|
+
def add(self, value):
|
|
90
|
+
self._data.add(value)
|
|
91
|
+
|
|
92
|
+
def has(self, value) -> bool:
|
|
93
|
+
return value in self._data
|
|
94
|
+
|
|
95
|
+
def delete(self, value):
|
|
96
|
+
self._data.discard(value)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def size(self) -> int:
|
|
100
|
+
return len(self._data)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def values(self) -> list:
|
|
104
|
+
return list(self._data)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class Map:
|
|
108
|
+
"""Key-value lookup. Transient storage, not for accumulation."""
|
|
109
|
+
|
|
110
|
+
__slots__ = ("_data",)
|
|
111
|
+
|
|
112
|
+
def __init__(self):
|
|
113
|
+
self._data = {}
|
|
114
|
+
|
|
115
|
+
def set(self, key: str, value):
|
|
116
|
+
self._data[key] = value
|
|
117
|
+
|
|
118
|
+
def get(self, key: str, default=None):
|
|
119
|
+
return self._data.get(key, default)
|
|
120
|
+
|
|
121
|
+
def has(self, key: str) -> bool:
|
|
122
|
+
return key in self._data
|
|
123
|
+
|
|
124
|
+
def delete(self, key: str):
|
|
125
|
+
self._data.pop(key, None)
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def keys(self) -> list:
|
|
129
|
+
return list(self._data.keys())
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def size(self) -> int:
|
|
133
|
+
return len(self._data)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class Buffer:
|
|
137
|
+
"""Ordered collection with optional size limit (sliding window)."""
|
|
138
|
+
|
|
139
|
+
__slots__ = ("_data", "_limit")
|
|
140
|
+
|
|
141
|
+
def __init__(self, limit: int | None = None):
|
|
142
|
+
self._data: deque = deque(maxlen=limit)
|
|
143
|
+
self._limit = limit
|
|
144
|
+
|
|
145
|
+
def push(self, item):
|
|
146
|
+
self._data.append(item)
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def items(self) -> list:
|
|
150
|
+
return list(self._data)
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def size(self) -> int:
|
|
154
|
+
return len(self._data)
|
|
155
|
+
|
|
156
|
+
def clear(self):
|
|
157
|
+
self._data.clear()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class Boolean:
|
|
161
|
+
"""Simple flag."""
|
|
162
|
+
|
|
163
|
+
__slots__ = ("_value",)
|
|
164
|
+
|
|
165
|
+
def __init__(self, initial: bool = False):
|
|
166
|
+
self._value = initial
|
|
167
|
+
|
|
168
|
+
def set(self, value: bool = True):
|
|
169
|
+
self._value = value
|
|
170
|
+
|
|
171
|
+
def toggle(self, value: bool | None = None):
|
|
172
|
+
if value is not None:
|
|
173
|
+
self._value = value
|
|
174
|
+
else:
|
|
175
|
+
self._value = not self._value
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def value(self) -> bool:
|
|
179
|
+
return self._value
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class BitMask:
|
|
183
|
+
"""Named flag set for tracking multiple boolean states."""
|
|
184
|
+
|
|
185
|
+
__slots__ = ("_flags",)
|
|
186
|
+
|
|
187
|
+
def __init__(self, names: list[str]):
|
|
188
|
+
self._flags = {name: False for name in names}
|
|
189
|
+
|
|
190
|
+
def set(self, name: str):
|
|
191
|
+
if name not in self._flags:
|
|
192
|
+
raise KeyError(f"Unknown flag: {name}")
|
|
193
|
+
self._flags[name] = True
|
|
194
|
+
|
|
195
|
+
def get(self, name: str) -> bool:
|
|
196
|
+
if name not in self._flags:
|
|
197
|
+
raise KeyError(f"Unknown flag: {name}")
|
|
198
|
+
return self._flags[name]
|
|
199
|
+
|
|
200
|
+
def clear(self, name: str):
|
|
201
|
+
if name not in self._flags:
|
|
202
|
+
raise KeyError(f"Unknown flag: {name}")
|
|
203
|
+
self._flags[name] = False
|
|
204
|
+
|
|
205
|
+
def all(self) -> bool:
|
|
206
|
+
return all(self._flags.values())
|
|
207
|
+
|
|
208
|
+
def any(self) -> bool:
|
|
209
|
+
return any(self._flags.values())
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class Trigger:
|
|
213
|
+
"""Condition checker across accumulators. Condition is a callable(memory) → bool."""
|
|
214
|
+
|
|
215
|
+
__slots__ = ("_condition", "_memory_ref")
|
|
216
|
+
|
|
217
|
+
def __init__(self, condition, memory):
|
|
218
|
+
self._condition = condition
|
|
219
|
+
self._memory_ref = memory
|
|
220
|
+
|
|
221
|
+
def check(self) -> bool:
|
|
222
|
+
return self._condition(self._memory_ref)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class Memory:
|
|
226
|
+
"""Shared memory — typed accumulator registry.
|
|
227
|
+
|
|
228
|
+
Accumulators are created on first access by name.
|
|
229
|
+
Same name always returns the same instance.
|
|
230
|
+
"""
|
|
231
|
+
|
|
232
|
+
def __init__(self):
|
|
233
|
+
self._store: dict = {}
|
|
234
|
+
|
|
235
|
+
def number(self, name: str, initial: float = 0) -> Number:
|
|
236
|
+
return self._get_or_create(name, Number, initial)
|
|
237
|
+
|
|
238
|
+
def average(self, name: str) -> Average:
|
|
239
|
+
return self._get_or_create(name, Average)
|
|
240
|
+
|
|
241
|
+
def minmax(self, name: str) -> MinMax:
|
|
242
|
+
return self._get_or_create(name, MinMax)
|
|
243
|
+
|
|
244
|
+
def set(self, name: str) -> Set:
|
|
245
|
+
return self._get_or_create(name, Set)
|
|
246
|
+
|
|
247
|
+
def map(self, name: str) -> Map:
|
|
248
|
+
return self._get_or_create(name, Map)
|
|
249
|
+
|
|
250
|
+
def buffer(self, name: str, limit: int | None = None) -> Buffer:
|
|
251
|
+
return self._get_or_create(name, Buffer, limit)
|
|
252
|
+
|
|
253
|
+
def boolean(self, name: str, initial: bool = False) -> Boolean:
|
|
254
|
+
return self._get_or_create(name, Boolean, initial)
|
|
255
|
+
|
|
256
|
+
def bitmask(self, name: str, flags: list[str]) -> BitMask:
|
|
257
|
+
return self._get_or_create(name, BitMask, flags)
|
|
258
|
+
|
|
259
|
+
def trigger(self, name: str, condition=None) -> Trigger:
|
|
260
|
+
if name in self._store:
|
|
261
|
+
return self._store[name]
|
|
262
|
+
if condition is None:
|
|
263
|
+
raise ValueError(f"Trigger '{name}' does not exist and no condition provided")
|
|
264
|
+
t = Trigger(condition, self)
|
|
265
|
+
self._store[name] = t
|
|
266
|
+
return t
|
|
267
|
+
|
|
268
|
+
def get(self, name: str):
|
|
269
|
+
"""Get accumulator by name. Returns None if not found."""
|
|
270
|
+
return self._store.get(name)
|
|
271
|
+
|
|
272
|
+
def delete(self, name: str):
|
|
273
|
+
"""Remove accumulator by name."""
|
|
274
|
+
self._store.pop(name, None)
|
|
275
|
+
|
|
276
|
+
def has(self, name: str) -> bool:
|
|
277
|
+
return name in self._store
|
|
278
|
+
|
|
279
|
+
def _get_or_create(self, name, cls, *args):
|
|
280
|
+
existing = self._store.get(name)
|
|
281
|
+
if existing is not None:
|
|
282
|
+
if not isinstance(existing, cls):
|
|
283
|
+
raise TypeError(
|
|
284
|
+
f"Accumulator '{name}' exists as {type(existing).__name__}, "
|
|
285
|
+
f"not {cls.__name__}"
|
|
286
|
+
)
|
|
287
|
+
return existing
|
|
288
|
+
instance = cls(*args)
|
|
289
|
+
self._store[name] = instance
|
|
290
|
+
return instance
|
dpe/_context.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Context object — passed to every processor invocation."""
|
|
2
|
+
|
|
3
|
+
from dpe._accumulators import Memory
|
|
4
|
+
from dpe._envelope import (
|
|
5
|
+
hash_file,
|
|
6
|
+
hash_string,
|
|
7
|
+
write_data,
|
|
8
|
+
write_error,
|
|
9
|
+
write_log,
|
|
10
|
+
write_meta,
|
|
11
|
+
write_stats,
|
|
12
|
+
write_trace,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Context:
|
|
17
|
+
"""Processing context created per invocation.
|
|
18
|
+
|
|
19
|
+
Provides framework API: output, emit, drain, meta, log, error, trace, stats, hash.
|
|
20
|
+
Each processor call gets its own ctx with current id/src.
|
|
21
|
+
Memory is shared across all processors in the tool.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
__slots__ = ("id", "src", "memory", "_runtime", "_labels")
|
|
25
|
+
|
|
26
|
+
def __init__(self, id: str, src: str, memory: Memory, runtime):
|
|
27
|
+
self.id = id
|
|
28
|
+
self.src = src
|
|
29
|
+
self.memory = memory
|
|
30
|
+
self._runtime = runtime
|
|
31
|
+
# Labels accumulated by ctx.trace(k, v). Flushed as one merged
|
|
32
|
+
# {type:"trace"} stderr event before each ctx.output(), then cleared.
|
|
33
|
+
self._labels: dict = {}
|
|
34
|
+
|
|
35
|
+
def trace(self, key: str, value):
|
|
36
|
+
"""Attach a label to this invocation's next output envelope.
|
|
37
|
+
|
|
38
|
+
Call as many times as needed between ctx.output() calls. The
|
|
39
|
+
accumulated labels are flushed once (merged) as a {type:"trace"}
|
|
40
|
+
event to stderr just before ctx.output() writes the envelope,
|
|
41
|
+
then cleared.
|
|
42
|
+
"""
|
|
43
|
+
self._labels[key] = value
|
|
44
|
+
|
|
45
|
+
def stats(self, **kwargs):
|
|
46
|
+
"""Emit a stats event to stderr. Example: ctx.stats(rows=1000, rps=250)."""
|
|
47
|
+
if kwargs:
|
|
48
|
+
write_stats(kwargs)
|
|
49
|
+
|
|
50
|
+
def output(self, v, *, id: str | None = None, src: str | None = None):
|
|
51
|
+
"""Emit data record to stdout.
|
|
52
|
+
|
|
53
|
+
Flushes accumulated trace labels as a merged trace event first,
|
|
54
|
+
then writes the envelope, then clears the label bag.
|
|
55
|
+
"""
|
|
56
|
+
out_id = id if id is not None else self.id
|
|
57
|
+
out_src = src if src is not None else self.src
|
|
58
|
+
# Emit trace even with empty labels — the chain row itself is the value.
|
|
59
|
+
write_trace(out_id, out_src, self._labels)
|
|
60
|
+
self._labels = {}
|
|
61
|
+
write_data(v, out_id, out_src)
|
|
62
|
+
|
|
63
|
+
def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
|
|
64
|
+
"""Emit to internal named queue. Processed by process_<queue>().
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
queue: Queue name (maps to process_<queue> function)
|
|
68
|
+
v: Payload data
|
|
69
|
+
id: Override id. Defaults to current ctx.id.
|
|
70
|
+
src: Override src. Defaults to current ctx.src.
|
|
71
|
+
"""
|
|
72
|
+
item_id = id if id is not None else self.id
|
|
73
|
+
item_src = src if src is not None else self.src
|
|
74
|
+
self._runtime.enqueue(queue, v, item_id, item_src)
|
|
75
|
+
|
|
76
|
+
def drain(self):
|
|
77
|
+
"""Block until all queued items are processed.
|
|
78
|
+
|
|
79
|
+
After drain returns, all accumulators reflect completed processing.
|
|
80
|
+
Can be called multiple times within one processor call.
|
|
81
|
+
"""
|
|
82
|
+
self._runtime.drain_queue()
|
|
83
|
+
|
|
84
|
+
def meta(self, v: dict):
|
|
85
|
+
"""Emit metadata record to stdout."""
|
|
86
|
+
write_meta(v)
|
|
87
|
+
|
|
88
|
+
def log(self, msg: str, *, level: str = "info", **extra):
|
|
89
|
+
"""Write structured log to stderr."""
|
|
90
|
+
write_log(msg, level, extra if extra else None)
|
|
91
|
+
|
|
92
|
+
def error(self, v, err):
|
|
93
|
+
"""Write error to stderr with original input preserved."""
|
|
94
|
+
write_error(v, err, self.id, self.src)
|
|
95
|
+
|
|
96
|
+
def hash(self, key: str) -> str:
|
|
97
|
+
"""Hash a string. Returns 16-char hex."""
|
|
98
|
+
return hash_string(key)
|
|
99
|
+
|
|
100
|
+
def hash_file(self, filepath: str, algorithm: str = "blake2b") -> str | None:
|
|
101
|
+
"""Hash file content in chunks. Returns hex string or None on error."""
|
|
102
|
+
return hash_file(filepath, algorithm)
|
dpe/_envelope.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""NDJSON envelope handling — parse, create, hash utilities."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def hash_string(key: str) -> str:
|
|
10
|
+
"""Hash a string. Returns 16-char hex (8 bytes blake2b)."""
|
|
11
|
+
return hashlib.blake2b(key.encode("utf-8"), digest_size=8).hexdigest()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def hash_file(filepath: str, algorithm: str = "blake2b", chunk_size: int = 65536) -> str | None:
|
|
15
|
+
"""Hash file content in chunks. Returns hex string or None on error.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
filepath: Path to file
|
|
19
|
+
algorithm: 'blake2b' (default) or 'xxhash' (requires xxhash package)
|
|
20
|
+
chunk_size: Read buffer size (default 64KB)
|
|
21
|
+
"""
|
|
22
|
+
# Type: hashlib.blake2b OR xxhash.xxh3_64. Both share .update(bytes)
|
|
23
|
+
# and .hexdigest() so we treat as Any to avoid a structural-protocol
|
|
24
|
+
# gymnastic for two short-circuit branches.
|
|
25
|
+
h: Any
|
|
26
|
+
try:
|
|
27
|
+
if algorithm == "xxhash":
|
|
28
|
+
try:
|
|
29
|
+
# xxhash is an optional runtime dep (see fallback). Need
|
|
30
|
+
# both codes here: `import-not-found` fires when xxhash isn't
|
|
31
|
+
# installed (CI), `unused-ignore` fires when it IS installed
|
|
32
|
+
# (some dev machines). Listing both keeps mypy happy in both
|
|
33
|
+
# environments without disabling the warning globally.
|
|
34
|
+
import xxhash # type: ignore[import-not-found,unused-ignore]
|
|
35
|
+
h = xxhash.xxh3_64()
|
|
36
|
+
except ImportError:
|
|
37
|
+
h = hashlib.blake2b(digest_size=16)
|
|
38
|
+
else:
|
|
39
|
+
h = hashlib.blake2b(digest_size=16)
|
|
40
|
+
|
|
41
|
+
with open(filepath, "rb") as f:
|
|
42
|
+
for chunk in iter(lambda: f.read(chunk_size), b""):
|
|
43
|
+
h.update(chunk)
|
|
44
|
+
return h.hexdigest()
|
|
45
|
+
except (PermissionError, FileNotFoundError, OSError):
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def compute_id(src: str, stage: str, v) -> str:
|
|
50
|
+
"""Compute envelope id from src + stage + v content."""
|
|
51
|
+
v_str = json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
|
52
|
+
key = f"{src}|{stage}|{v_str}"
|
|
53
|
+
return hash_string(key)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def parse_envelope(line: str) -> dict | None:
|
|
57
|
+
"""Parse a single NDJSON line into envelope dict. Returns None on parse error."""
|
|
58
|
+
line = line.strip()
|
|
59
|
+
if not line:
|
|
60
|
+
return None
|
|
61
|
+
try:
|
|
62
|
+
return json.loads(line)
|
|
63
|
+
except json.JSONDecodeError:
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def write_data(v, id: str, src: str):
|
|
68
|
+
"""Write a data envelope to stdout."""
|
|
69
|
+
_write_stdout({"t": "d", "id": id, "src": src, "v": v})
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def write_meta(v: dict):
|
|
73
|
+
"""Write a metadata envelope to stdout."""
|
|
74
|
+
_write_stdout({"t": "m", "v": v})
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def write_log(msg: str, level: str, extra: dict | None = None):
|
|
78
|
+
"""Write structured log to stderr."""
|
|
79
|
+
record: dict = {"type": "log", "level": level, "msg": msg}
|
|
80
|
+
if extra:
|
|
81
|
+
record.update(extra)
|
|
82
|
+
_write_stderr(record)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def write_error(v, err, id: str, src: str):
|
|
86
|
+
"""Write error to stderr with original input preserved."""
|
|
87
|
+
record = {
|
|
88
|
+
"type": "error",
|
|
89
|
+
"error": str(err),
|
|
90
|
+
"input": v,
|
|
91
|
+
"id": id,
|
|
92
|
+
"src": src,
|
|
93
|
+
}
|
|
94
|
+
_write_stderr(record)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def write_trace(id: str, src: str, labels: dict):
|
|
98
|
+
"""Write merged trace event to stderr. Called by ctx before each output."""
|
|
99
|
+
_write_stderr({"type": "trace", "id": id, "src": src, "labels": labels})
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def write_stats(data: dict):
|
|
103
|
+
"""Write a stats event to stderr."""
|
|
104
|
+
record = {"type": "stats"}
|
|
105
|
+
record.update(data)
|
|
106
|
+
_write_stderr(record)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _write_stdout(obj: dict):
|
|
110
|
+
sys.stdout.write(json.dumps(obj, ensure_ascii=False, separators=(",", ":")) + "\n")
|
|
111
|
+
sys.stdout.flush()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _write_stderr(obj: dict):
|
|
115
|
+
sys.stderr.write(json.dumps(obj, ensure_ascii=False, separators=(",", ":")) + "\n")
|
|
116
|
+
sys.stderr.flush()
|
dpe/_runtime.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Runtime — main loop, discovery, queue management, signal handling."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import json
|
|
5
|
+
import signal
|
|
6
|
+
import sys
|
|
7
|
+
from collections import deque
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
|
|
10
|
+
from dpe._accumulators import Memory
|
|
11
|
+
from dpe._context import Context
|
|
12
|
+
from dpe._envelope import parse_envelope, write_log
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Runtime:
|
|
16
|
+
"""Core runtime engine. Manages stdin loop, queue, and processor dispatch."""
|
|
17
|
+
|
|
18
|
+
def __init__(self):
|
|
19
|
+
self.settings: dict = {}
|
|
20
|
+
self.memory = Memory()
|
|
21
|
+
self.input_fn: Callable | None = None
|
|
22
|
+
self.processors: dict[str, Callable] = {}
|
|
23
|
+
self._queue: deque = deque()
|
|
24
|
+
self._shutdown = False
|
|
25
|
+
|
|
26
|
+
def enqueue(self, queue_name: str, v, id: str, src: str):
|
|
27
|
+
"""Add item to internal queue."""
|
|
28
|
+
self._queue.append((queue_name, v, id, src))
|
|
29
|
+
|
|
30
|
+
def drain_queue(self):
|
|
31
|
+
"""Process all queued items until empty. Queue processors may emit more items."""
|
|
32
|
+
max_iterations = 100_000
|
|
33
|
+
iterations = 0
|
|
34
|
+
|
|
35
|
+
while self._queue and iterations < max_iterations:
|
|
36
|
+
name, v, item_id, item_src = self._queue.popleft()
|
|
37
|
+
|
|
38
|
+
proc = self.processors.get(name)
|
|
39
|
+
if proc is None:
|
|
40
|
+
write_log(f"No processor for queue '{name}', dropping item", "warn")
|
|
41
|
+
continue
|
|
42
|
+
|
|
43
|
+
ctx = Context(item_id, item_src, self.memory, self)
|
|
44
|
+
try:
|
|
45
|
+
proc(v, self.settings, ctx)
|
|
46
|
+
except Exception as e:
|
|
47
|
+
ctx.error(v, e)
|
|
48
|
+
|
|
49
|
+
iterations += 1
|
|
50
|
+
|
|
51
|
+
if iterations >= max_iterations:
|
|
52
|
+
write_log(f"Queue drain hit safety limit ({max_iterations})", "error")
|
|
53
|
+
|
|
54
|
+
def create_ctx(self, id: str, src: str) -> Context:
|
|
55
|
+
return Context(id, src, self.memory, self)
|
|
56
|
+
|
|
57
|
+
def _handle_signal(self, signum, frame):
|
|
58
|
+
self._shutdown = True
|
|
59
|
+
|
|
60
|
+
def run(self, caller_module):
|
|
61
|
+
"""Main entry point. Discovers processors, reads stdin, dispatches."""
|
|
62
|
+
# Register signal handlers
|
|
63
|
+
signal.signal(signal.SIGTERM, self._handle_signal)
|
|
64
|
+
signal.signal(signal.SIGINT, self._handle_signal)
|
|
65
|
+
|
|
66
|
+
# Discover processors
|
|
67
|
+
self.input_fn, self.processors = _discover_processors(caller_module)
|
|
68
|
+
if self.input_fn is None:
|
|
69
|
+
write_log("No process_input function found in tool module", "error")
|
|
70
|
+
sys.exit(1)
|
|
71
|
+
|
|
72
|
+
# Parse settings
|
|
73
|
+
self.settings = _parse_settings()
|
|
74
|
+
|
|
75
|
+
# Main stdin loop
|
|
76
|
+
for line in sys.stdin:
|
|
77
|
+
if self._shutdown:
|
|
78
|
+
break
|
|
79
|
+
|
|
80
|
+
envelope = parse_envelope(line)
|
|
81
|
+
if envelope is None:
|
|
82
|
+
continue
|
|
83
|
+
|
|
84
|
+
# Skip non-data lines
|
|
85
|
+
if envelope.get("t") != "d":
|
|
86
|
+
continue
|
|
87
|
+
|
|
88
|
+
id = envelope.get("id", "")
|
|
89
|
+
src = envelope.get("src", "")
|
|
90
|
+
v = envelope.get("v", {})
|
|
91
|
+
|
|
92
|
+
ctx = self.create_ctx(id, src)
|
|
93
|
+
try:
|
|
94
|
+
self.input_fn(v, self.settings, ctx)
|
|
95
|
+
except Exception as e:
|
|
96
|
+
ctx.error(v, e)
|
|
97
|
+
|
|
98
|
+
# Auto-drain if queue has items (process_input didn't call drain)
|
|
99
|
+
if self._queue:
|
|
100
|
+
self.drain_queue()
|
|
101
|
+
|
|
102
|
+
# stdin EOF or SIGTERM: drain remaining queue
|
|
103
|
+
if self._queue:
|
|
104
|
+
self.drain_queue()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _discover_processors(module) -> tuple[Callable | None, dict[str, Callable]]:
|
|
108
|
+
"""Find process_input and process_<name> functions in the tool module."""
|
|
109
|
+
input_fn: Callable | None = None
|
|
110
|
+
queue_fns: dict[str, Callable] = {}
|
|
111
|
+
|
|
112
|
+
for name, obj in inspect.getmembers(module, inspect.isfunction):
|
|
113
|
+
if not name.startswith("process_"):
|
|
114
|
+
continue
|
|
115
|
+
# Only discover functions defined in the module itself, not imports
|
|
116
|
+
if getattr(obj, "__module__", None) != module.__name__:
|
|
117
|
+
continue
|
|
118
|
+
|
|
119
|
+
suffix = name[len("process_"):]
|
|
120
|
+
if suffix == "input":
|
|
121
|
+
input_fn = obj
|
|
122
|
+
else:
|
|
123
|
+
queue_fns[suffix] = obj
|
|
124
|
+
|
|
125
|
+
return input_fn, queue_fns
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _parse_settings() -> dict:
|
|
129
|
+
"""Parse settings from argv[1] JSON string."""
|
|
130
|
+
if len(sys.argv) < 2:
|
|
131
|
+
return {}
|
|
132
|
+
try:
|
|
133
|
+
return json.loads(sys.argv[1])
|
|
134
|
+
except (json.JSONDecodeError, IndexError):
|
|
135
|
+
write_log("Failed to parse settings from argv[1], using empty dict", "warn")
|
|
136
|
+
return {}
|