combycode-dpe 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,6 @@
1
+ __pycache__/
2
+ *.pyc
3
+ *.egg-info/
4
+ dist/
5
+ build/
6
+ .pytest_cache/
@@ -0,0 +1,11 @@
1
+ Metadata-Version: 2.4
2
+ Name: combycode-dpe
3
+ Version: 2.0.0
4
+ Summary: DPE framework for building streaming pipeline tools in Python
5
+ Project-URL: Repository, https://github.com/combycode/dpe-framework-python
6
+ License-Expression: AGPL-3.0-or-later
7
+ Requires-Python: >=3.11
8
+ Provides-Extra: dev
9
+ Requires-Dist: jsonschema>=4.20; extra == 'dev'
10
+ Requires-Dist: mypy>=1.10; extra == 'dev'
11
+ Requires-Dist: pytest>=8.0; extra == 'dev'
@@ -0,0 +1,107 @@
1
+ # combycode-dpe (Python)
2
+
3
+ DPE framework for building streaming pipeline tools in Python.
4
+
5
+ A DPE tool is an independent program: it reads NDJSON envelopes from stdin,
6
+ processes each one, and writes NDJSON envelopes to stdout. The runner
7
+ (`dpe`) spawns it, pipes data through it, and routes its output to the
8
+ next stage. The framework handles the loop, the parsing, the queues, and
9
+ the error/trace plumbing — your code only writes the transform.
10
+
11
+ Requires Python 3.11+.
12
+
13
+ ## Install
14
+
15
+ ```bash
16
+ pip install combycode-dpe
17
+ # or in a per-tool venv:
18
+ uv venv --seed .venv && .venv/bin/pip install combycode-dpe
19
+ ```
20
+
21
+ ## Hello tool
22
+
23
+ ```python
24
+ # src/my_tool/main.py
25
+ import dpe
26
+
27
+ def process_input(v, settings, ctx):
28
+ ctx.output(v) # pass-through
29
+
30
+ if __name__ == "__main__":
31
+ dpe.run()
32
+ ```
33
+
34
+ Run as a tool: `python src/my_tool/main.py '{"...settings JSON..."}'`. The
35
+ runner passes settings as `argv[1]` and pipes NDJSON through stdin/stdout.
36
+
37
+ `dpe.run()` introspects the calling module to find `process_input` and
38
+ any `process_<queue>` handlers — no manual registration needed.
39
+
40
+ ## Processor signature
41
+
42
+ ```python
43
+ def process_input(v, settings, ctx):
44
+ ...
45
+ ```
46
+
47
+ - `v` — envelope payload (a dict / list / scalar — whatever JSON parsed to).
48
+ - `settings` — argv[1] parsed once at startup.
49
+ - `ctx` — emits output, errors, logs, metadata; reads/writes shared
50
+ accumulators in `ctx.memory`.
51
+
52
+ ## Internal queues
53
+
54
+ ```python
55
+ import dpe
56
+
57
+ def process_input(v, settings, ctx):
58
+ ctx.emit("validate", v)
59
+
60
+ def process_validate(v, settings, ctx):
61
+ if v.get("ok"):
62
+ ctx.output(v)
63
+ else:
64
+ ctx.error(v, "validation failed")
65
+
66
+ if __name__ == "__main__":
67
+ dpe.run()
68
+ ```
69
+
70
+ `ctx.emit(queue, v)` enqueues for the named handler. `ctx.drain()` blocks
71
+ until every queue is empty.
72
+
73
+ ## Context API (essentials)
74
+
75
+ | Call | What it does |
76
+ |---|---|
77
+ | `ctx.output(v, id=None, src=None)` | Emit `{t:"d",...}` to stdout. `id`/`src` default to inherit. |
78
+ | `ctx.emit(queue, v)` | Push to internal queue, processed by `process_<queue>`. |
79
+ | `ctx.drain()` | Block until all queues are empty. |
80
+ | `ctx.error(v, msg)` | Write `{type:"error",input:v,error:msg,...}` to stderr. |
81
+ | `ctx.log(level, msg)` | Structured stderr log. |
82
+ | `ctx.meta(v)` | Emit `{t:"m", v}` envelope to stdout. |
83
+ | `ctx.memory` | Typed accumulators: `Number`, `Average`, `MinMax`, `Set`, `Map`, `Buffer`, `Boolean`, `BitMask`, `Trigger`. |
84
+
85
+ See `docs/frameworks.md` in the monorepo for the full reference.
86
+
87
+ ## Scaffolding
88
+
89
+ ```bash
90
+ dpe-dev scaffold --name my-tool --runtime python --out ./my-tool --description "what it does"
91
+ cd my-tool
92
+ dpe-dev build . # creates .venv, installs the tool with [dev] extras
93
+ dpe-dev test . # pytest inside the venv
94
+ dpe-dev verify . # spawn the tool's main.py, feed input.ndjson, diff stdout vs expected.ndjson
95
+ ```
96
+
97
+ ## Optional speed-up
98
+
99
+ For bulk-hashing payloads, install `xxhash` and pass `algorithm="xxhash"`
100
+ to `dpe.hash_file()`. The framework falls back to `hashlib.blake2b` when
101
+ xxhash is missing.
102
+
103
+ ## Repo & licence
104
+
105
+ - Source: <https://github.com/combycode/dpe-framework-python>
106
+ - Monorepo (canonical): <https://github.com/combycode/dpe>
107
+ - Licence: AGPL-3.0-or-later
@@ -0,0 +1,51 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "combycode-dpe"
7
+ version = "2.0.0"
8
+ description = "DPE framework for building streaming pipeline tools in Python"
9
+ license = "AGPL-3.0-or-later"
10
+ requires-python = ">=3.11"
11
+ dependencies = []
12
+
13
+ [project.optional-dependencies]
14
+ dev = ["pytest>=8.0", "jsonschema>=4.20", "mypy>=1.10"]
15
+
16
+ [project.urls]
17
+ Repository = "https://github.com/combycode/dpe-framework-python"
18
+
19
+ [tool.hatch.build.targets.wheel]
20
+ packages = ["src/dpe"]
21
+
22
+ [tool.ruff]
23
+ line-length = 100
24
+ target-version = "py311"
25
+ extend-exclude = ["template"]
26
+
27
+ [tool.ruff.lint]
28
+ select = [
29
+ "E", # pycodestyle errors
30
+ "F", # pyflakes
31
+ "W", # pycodestyle warnings
32
+ "I", # isort
33
+ "B", # flake8-bugbear
34
+ "UP", # pyupgrade
35
+ ]
36
+ ignore = []
37
+
38
+ [tool.ruff.format]
39
+ quote-style = "double"
40
+ indent-style = "space"
41
+
42
+ [tool.mypy]
43
+ python_version = "3.11"
44
+ files = ["src/dpe"]
45
+ # Pragmatic strictness: catch real type bugs without forcing every helper
46
+ # to carry full annotations. Bump to `strict = true` in a follow-up pass
47
+ # when the codebase is fully annotated.
48
+ no_implicit_optional = true
49
+ warn_redundant_casts = true
50
+ warn_unused_ignores = true
51
+ check_untyped_defs = false
@@ -0,0 +1,76 @@
1
+ """
2
+ DPE — Data Processing Engine framework.
3
+
4
+ Build streaming pipeline tools with minimal boilerplate.
5
+
6
+ Usage:
7
+ import dpe
8
+
9
+ def process_input(v, settings, ctx):
10
+ ctx.output({"result": transform(v)})
11
+ ctx.emit("validate", {"data": v})
12
+
13
+ def process_validate(v, settings, ctx):
14
+ if ok(v):
15
+ ctx.output(v)
16
+ else:
17
+ ctx.error(v, "validation failed")
18
+
19
+ if __name__ == "__main__":
20
+ dpe.run()
21
+ """
22
+
23
+ import inspect
24
+
25
+ from dpe._accumulators import (
26
+ Average,
27
+ BitMask,
28
+ Boolean,
29
+ Buffer,
30
+ Map,
31
+ Memory,
32
+ MinMax,
33
+ Number,
34
+ Set,
35
+ Trigger,
36
+ )
37
+ from dpe._context import Context
38
+ from dpe._envelope import hash_file, hash_string
39
+ from dpe._runtime import Runtime
40
+
41
+
42
+ def run():
43
+ """Start the tool processing loop.
44
+
45
+ Discovers process_input and process_<queue> functions in the calling module,
46
+ parses settings from argv[1], reads stdin line by line, and dispatches.
47
+
48
+ Call this at the bottom of your tool:
49
+ if __name__ == "__main__":
50
+ dpe.run()
51
+ """
52
+ frame = inspect.stack()[1]
53
+ caller_module = inspect.getmodule(frame[0])
54
+ if caller_module is None:
55
+ raise RuntimeError("Cannot discover caller module")
56
+
57
+ runtime = Runtime()
58
+ runtime.run(caller_module)
59
+
60
+
61
+ __all__ = [
62
+ "run",
63
+ "hash_string",
64
+ "hash_file",
65
+ "Context",
66
+ "Memory",
67
+ "Number",
68
+ "Average",
69
+ "MinMax",
70
+ "Set",
71
+ "Map",
72
+ "Buffer",
73
+ "Boolean",
74
+ "BitMask",
75
+ "Trigger",
76
+ ]
@@ -0,0 +1,290 @@
1
+ """Typed accumulators for shared memory across processors."""
2
+
3
+ from collections import deque
4
+
5
+
6
+ class Number:
7
+ """Numeric accumulator with increment/decrement."""
8
+
9
+ __slots__ = ("_value",)
10
+
11
+ def __init__(self, initial: float = 0):
12
+ self._value = initial
13
+
14
+ def inc(self, delta: float = 1) -> float:
15
+ self._value += delta
16
+ return self._value
17
+
18
+ def dec(self, delta: float = 1) -> float:
19
+ self._value -= delta
20
+ return self._value
21
+
22
+ @property
23
+ def value(self) -> float:
24
+ return self._value
25
+
26
+
27
+ class Average:
28
+ """Running average without storing all values."""
29
+
30
+ __slots__ = ("_sum", "_count")
31
+
32
+ def __init__(self):
33
+ self._sum = 0.0
34
+ self._count = 0
35
+
36
+ def add(self, value: float):
37
+ self._sum += value
38
+ self._count += 1
39
+
40
+ @property
41
+ def value(self) -> float:
42
+ if self._count == 0:
43
+ return 0.0
44
+ return self._sum / self._count
45
+
46
+ @property
47
+ def count(self) -> int:
48
+ return self._count
49
+
50
+
51
+ class MinMax:
52
+ """Track minimum and maximum values."""
53
+
54
+ __slots__ = ("_min", "_max", "_count")
55
+
56
+ def __init__(self):
57
+ self._min = None
58
+ self._max = None
59
+ self._count = 0
60
+
61
+ def add(self, value: float):
62
+ if self._min is None or value < self._min:
63
+ self._min = value
64
+ if self._max is None or value > self._max:
65
+ self._max = value
66
+ self._count += 1
67
+
68
+ @property
69
+ def min(self) -> float | None:
70
+ return self._min
71
+
72
+ @property
73
+ def max(self) -> float | None:
74
+ return self._max
75
+
76
+ @property
77
+ def count(self) -> int:
78
+ return self._count
79
+
80
+
81
+ class Set:
82
+ """Unique value collection."""
83
+
84
+ __slots__ = ("_data",)
85
+
86
+ def __init__(self):
87
+ self._data = set()
88
+
89
+ def add(self, value):
90
+ self._data.add(value)
91
+
92
+ def has(self, value) -> bool:
93
+ return value in self._data
94
+
95
+ def delete(self, value):
96
+ self._data.discard(value)
97
+
98
+ @property
99
+ def size(self) -> int:
100
+ return len(self._data)
101
+
102
+ @property
103
+ def values(self) -> list:
104
+ return list(self._data)
105
+
106
+
107
+ class Map:
108
+ """Key-value lookup. Transient storage, not for accumulation."""
109
+
110
+ __slots__ = ("_data",)
111
+
112
+ def __init__(self):
113
+ self._data = {}
114
+
115
+ def set(self, key: str, value):
116
+ self._data[key] = value
117
+
118
+ def get(self, key: str, default=None):
119
+ return self._data.get(key, default)
120
+
121
+ def has(self, key: str) -> bool:
122
+ return key in self._data
123
+
124
+ def delete(self, key: str):
125
+ self._data.pop(key, None)
126
+
127
+ @property
128
+ def keys(self) -> list:
129
+ return list(self._data.keys())
130
+
131
+ @property
132
+ def size(self) -> int:
133
+ return len(self._data)
134
+
135
+
136
+ class Buffer:
137
+ """Ordered collection with optional size limit (sliding window)."""
138
+
139
+ __slots__ = ("_data", "_limit")
140
+
141
+ def __init__(self, limit: int | None = None):
142
+ self._data: deque = deque(maxlen=limit)
143
+ self._limit = limit
144
+
145
+ def push(self, item):
146
+ self._data.append(item)
147
+
148
+ @property
149
+ def items(self) -> list:
150
+ return list(self._data)
151
+
152
+ @property
153
+ def size(self) -> int:
154
+ return len(self._data)
155
+
156
+ def clear(self):
157
+ self._data.clear()
158
+
159
+
160
+ class Boolean:
161
+ """Simple flag."""
162
+
163
+ __slots__ = ("_value",)
164
+
165
+ def __init__(self, initial: bool = False):
166
+ self._value = initial
167
+
168
+ def set(self, value: bool = True):
169
+ self._value = value
170
+
171
+ def toggle(self, value: bool | None = None):
172
+ if value is not None:
173
+ self._value = value
174
+ else:
175
+ self._value = not self._value
176
+
177
+ @property
178
+ def value(self) -> bool:
179
+ return self._value
180
+
181
+
182
+ class BitMask:
183
+ """Named flag set for tracking multiple boolean states."""
184
+
185
+ __slots__ = ("_flags",)
186
+
187
+ def __init__(self, names: list[str]):
188
+ self._flags = {name: False for name in names}
189
+
190
+ def set(self, name: str):
191
+ if name not in self._flags:
192
+ raise KeyError(f"Unknown flag: {name}")
193
+ self._flags[name] = True
194
+
195
+ def get(self, name: str) -> bool:
196
+ if name not in self._flags:
197
+ raise KeyError(f"Unknown flag: {name}")
198
+ return self._flags[name]
199
+
200
+ def clear(self, name: str):
201
+ if name not in self._flags:
202
+ raise KeyError(f"Unknown flag: {name}")
203
+ self._flags[name] = False
204
+
205
+ def all(self) -> bool:
206
+ return all(self._flags.values())
207
+
208
+ def any(self) -> bool:
209
+ return any(self._flags.values())
210
+
211
+
212
+ class Trigger:
213
+ """Condition checker across accumulators. Condition is a callable(memory) → bool."""
214
+
215
+ __slots__ = ("_condition", "_memory_ref")
216
+
217
+ def __init__(self, condition, memory):
218
+ self._condition = condition
219
+ self._memory_ref = memory
220
+
221
+ def check(self) -> bool:
222
+ return self._condition(self._memory_ref)
223
+
224
+
225
+ class Memory:
226
+ """Shared memory — typed accumulator registry.
227
+
228
+ Accumulators are created on first access by name.
229
+ Same name always returns the same instance.
230
+ """
231
+
232
+ def __init__(self):
233
+ self._store: dict = {}
234
+
235
+ def number(self, name: str, initial: float = 0) -> Number:
236
+ return self._get_or_create(name, Number, initial)
237
+
238
+ def average(self, name: str) -> Average:
239
+ return self._get_or_create(name, Average)
240
+
241
+ def minmax(self, name: str) -> MinMax:
242
+ return self._get_or_create(name, MinMax)
243
+
244
+ def set(self, name: str) -> Set:
245
+ return self._get_or_create(name, Set)
246
+
247
+ def map(self, name: str) -> Map:
248
+ return self._get_or_create(name, Map)
249
+
250
+ def buffer(self, name: str, limit: int | None = None) -> Buffer:
251
+ return self._get_or_create(name, Buffer, limit)
252
+
253
+ def boolean(self, name: str, initial: bool = False) -> Boolean:
254
+ return self._get_or_create(name, Boolean, initial)
255
+
256
+ def bitmask(self, name: str, flags: list[str]) -> BitMask:
257
+ return self._get_or_create(name, BitMask, flags)
258
+
259
+ def trigger(self, name: str, condition=None) -> Trigger:
260
+ if name in self._store:
261
+ return self._store[name]
262
+ if condition is None:
263
+ raise ValueError(f"Trigger '{name}' does not exist and no condition provided")
264
+ t = Trigger(condition, self)
265
+ self._store[name] = t
266
+ return t
267
+
268
+ def get(self, name: str):
269
+ """Get accumulator by name. Returns None if not found."""
270
+ return self._store.get(name)
271
+
272
+ def delete(self, name: str):
273
+ """Remove accumulator by name."""
274
+ self._store.pop(name, None)
275
+
276
+ def has(self, name: str) -> bool:
277
+ return name in self._store
278
+
279
+ def _get_or_create(self, name, cls, *args):
280
+ existing = self._store.get(name)
281
+ if existing is not None:
282
+ if not isinstance(existing, cls):
283
+ raise TypeError(
284
+ f"Accumulator '{name}' exists as {type(existing).__name__}, "
285
+ f"not {cls.__name__}"
286
+ )
287
+ return existing
288
+ instance = cls(*args)
289
+ self._store[name] = instance
290
+ return instance
@@ -0,0 +1,102 @@
1
+ """Context object — passed to every processor invocation."""
2
+
3
+ from dpe._accumulators import Memory
4
+ from dpe._envelope import (
5
+ hash_file,
6
+ hash_string,
7
+ write_data,
8
+ write_error,
9
+ write_log,
10
+ write_meta,
11
+ write_stats,
12
+ write_trace,
13
+ )
14
+
15
+
16
+ class Context:
17
+ """Processing context created per invocation.
18
+
19
+ Provides framework API: output, emit, drain, meta, log, error, trace, stats, hash.
20
+ Each processor call gets its own ctx with current id/src.
21
+ Memory is shared across all processors in the tool.
22
+ """
23
+
24
+ __slots__ = ("id", "src", "memory", "_runtime", "_labels")
25
+
26
+ def __init__(self, id: str, src: str, memory: Memory, runtime):
27
+ self.id = id
28
+ self.src = src
29
+ self.memory = memory
30
+ self._runtime = runtime
31
+ # Labels accumulated by ctx.trace(k, v). Flushed as one merged
32
+ # {type:"trace"} stderr event before each ctx.output(), then cleared.
33
+ self._labels: dict = {}
34
+
35
+ def trace(self, key: str, value):
36
+ """Attach a label to this invocation's next output envelope.
37
+
38
+ Call as many times as needed between ctx.output() calls. The
39
+ accumulated labels are flushed once (merged) as a {type:"trace"}
40
+ event to stderr just before ctx.output() writes the envelope,
41
+ then cleared.
42
+ """
43
+ self._labels[key] = value
44
+
45
+ def stats(self, **kwargs):
46
+ """Emit a stats event to stderr. Example: ctx.stats(rows=1000, rps=250)."""
47
+ if kwargs:
48
+ write_stats(kwargs)
49
+
50
+ def output(self, v, *, id: str | None = None, src: str | None = None):
51
+ """Emit data record to stdout.
52
+
53
+ Flushes accumulated trace labels as a merged trace event first,
54
+ then writes the envelope, then clears the label bag.
55
+ """
56
+ out_id = id if id is not None else self.id
57
+ out_src = src if src is not None else self.src
58
+ # Emit trace even with empty labels — the chain row itself is the value.
59
+ write_trace(out_id, out_src, self._labels)
60
+ self._labels = {}
61
+ write_data(v, out_id, out_src)
62
+
63
+ def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
64
+ """Emit to internal named queue. Processed by process_<queue>().
65
+
66
+ Args:
67
+ queue: Queue name (maps to process_<queue> function)
68
+ v: Payload data
69
+ id: Override id. Defaults to current ctx.id.
70
+ src: Override src. Defaults to current ctx.src.
71
+ """
72
+ item_id = id if id is not None else self.id
73
+ item_src = src if src is not None else self.src
74
+ self._runtime.enqueue(queue, v, item_id, item_src)
75
+
76
+ def drain(self):
77
+ """Block until all queued items are processed.
78
+
79
+ After drain returns, all accumulators reflect completed processing.
80
+ Can be called multiple times within one processor call.
81
+ """
82
+ self._runtime.drain_queue()
83
+
84
+ def meta(self, v: dict):
85
+ """Emit metadata record to stdout."""
86
+ write_meta(v)
87
+
88
+ def log(self, msg: str, *, level: str = "info", **extra):
89
+ """Write structured log to stderr."""
90
+ write_log(msg, level, extra if extra else None)
91
+
92
+ def error(self, v, err):
93
+ """Write error to stderr with original input preserved."""
94
+ write_error(v, err, self.id, self.src)
95
+
96
+ def hash(self, key: str) -> str:
97
+ """Hash a string. Returns 16-char hex."""
98
+ return hash_string(key)
99
+
100
+ def hash_file(self, filepath: str, algorithm: str = "blake2b") -> str | None:
101
+ """Hash file content in chunks. Returns hex string or None on error."""
102
+ return hash_file(filepath, algorithm)