combycode-dpe 2.0.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of combycode-dpe might be problematic. Click here for more details.
- combycode_dpe-2.0.0rc1/.gitignore +6 -0
- combycode_dpe-2.0.0rc1/PKG-INFO +11 -0
- combycode_dpe-2.0.0rc1/README.md +107 -0
- combycode_dpe-2.0.0rc1/pyproject.toml +51 -0
- combycode_dpe-2.0.0rc1/src/dpe/__init__.py +76 -0
- combycode_dpe-2.0.0rc1/src/dpe/_accumulators.py +290 -0
- combycode_dpe-2.0.0rc1/src/dpe/_context.py +102 -0
- combycode_dpe-2.0.0rc1/src/dpe/_envelope.py +111 -0
- combycode_dpe-2.0.0rc1/src/dpe/_runtime.py +136 -0
- combycode_dpe-2.0.0rc1/template/meta.json +10 -0
- combycode_dpe-2.0.0rc1/template/pyproject.toml +36 -0
- combycode_dpe-2.0.0rc1/template/spec.yaml +25 -0
- combycode_dpe-2.0.0rc1/template/src/__PKG__/__init__.py +0 -0
- combycode_dpe-2.0.0rc1/template/src/__PKG__/main.py +27 -0
- combycode_dpe-2.0.0rc1/template/tests/test_basic.py +10 -0
- combycode_dpe-2.0.0rc1/template/verify/case-basic/expected.ndjson +2 -0
- combycode_dpe-2.0.0rc1/template/verify/case-basic/input.ndjson +2 -0
- combycode_dpe-2.0.0rc1/template/verify/case-basic/settings.json +1 -0
- combycode_dpe-2.0.0rc1/tests/test_accumulators.py +376 -0
- combycode_dpe-2.0.0rc1/tests/test_integration.py +548 -0
- combycode_dpe-2.0.0rc1/tests/test_stderr_events_conformance.py +122 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: combycode-dpe
|
|
3
|
+
Version: 2.0.0rc1
|
|
4
|
+
Summary: DPE framework for building streaming pipeline tools in Python
|
|
5
|
+
Project-URL: Repository, https://github.com/combycode/dpe-framework-python
|
|
6
|
+
License-Expression: AGPL-3.0-or-later
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
Requires-Dist: jsonschema>=4.20; extra == 'dev'
|
|
10
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
11
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# combycode-dpe (Python)
|
|
2
|
+
|
|
3
|
+
DPE framework for building streaming pipeline tools in Python.
|
|
4
|
+
|
|
5
|
+
A DPE tool is an independent program: it reads NDJSON envelopes from stdin,
|
|
6
|
+
processes each one, and writes NDJSON envelopes to stdout. The runner
|
|
7
|
+
(`dpe`) spawns it, pipes data through it, and routes its output to the
|
|
8
|
+
next stage. The framework handles the loop, the parsing, the queues, and
|
|
9
|
+
the error/trace plumbing — your code only writes the transform.
|
|
10
|
+
|
|
11
|
+
Requires Python 3.11+.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install combycode-dpe
|
|
17
|
+
# or in a per-tool venv:
|
|
18
|
+
uv venv --seed .venv && .venv/bin/pip install combycode-dpe
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Hello tool
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
# src/my_tool/main.py
|
|
25
|
+
import dpe
|
|
26
|
+
|
|
27
|
+
def process_input(v, settings, ctx):
|
|
28
|
+
ctx.output(v) # pass-through
|
|
29
|
+
|
|
30
|
+
if __name__ == "__main__":
|
|
31
|
+
dpe.run()
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Run as a tool: `python src/my_tool/main.py '{"...settings JSON..."}'`. The
|
|
35
|
+
runner passes settings as `argv[1]` and pipes NDJSON through stdin/stdout.
|
|
36
|
+
|
|
37
|
+
`dpe.run()` introspects the calling module to find `process_input` and
|
|
38
|
+
any `process_<queue>` handlers — no manual registration needed.
|
|
39
|
+
|
|
40
|
+
## Processor signature
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
def process_input(v, settings, ctx):
|
|
44
|
+
...
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
- `v` — envelope payload (a dict / list / scalar — whatever JSON parsed to).
|
|
48
|
+
- `settings` — argv[1] parsed once at startup.
|
|
49
|
+
- `ctx` — emits output, errors, logs, metadata; reads/writes shared
|
|
50
|
+
accumulators in `ctx.memory`.
|
|
51
|
+
|
|
52
|
+
## Internal queues
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
import dpe
|
|
56
|
+
|
|
57
|
+
def process_input(v, settings, ctx):
|
|
58
|
+
ctx.emit("validate", v)
|
|
59
|
+
|
|
60
|
+
def process_validate(v, settings, ctx):
|
|
61
|
+
if v.get("ok"):
|
|
62
|
+
ctx.output(v)
|
|
63
|
+
else:
|
|
64
|
+
ctx.error(v, "validation failed")
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
dpe.run()
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`ctx.emit(queue, v)` enqueues for the named handler. `ctx.drain()` blocks
|
|
71
|
+
until every queue is empty.
|
|
72
|
+
|
|
73
|
+
## Context API (essentials)
|
|
74
|
+
|
|
75
|
+
| Call | What it does |
|
|
76
|
+
|---|---|
|
|
77
|
+
| `ctx.output(v, id=None, src=None)` | Emit `{t:"d",...}` to stdout. `id`/`src` default to inherit. |
|
|
78
|
+
| `ctx.emit(queue, v)` | Push to internal queue, processed by `process_<queue>`. |
|
|
79
|
+
| `ctx.drain()` | Block until all queues are empty. |
|
|
80
|
+
| `ctx.error(v, msg)` | Write `{type:"error",input:v,error:msg,...}` to stderr. |
|
|
81
|
+
| `ctx.log(level, msg)` | Structured stderr log. |
|
|
82
|
+
| `ctx.meta(v)` | Emit `{t:"m", v}` envelope to stdout. |
|
|
83
|
+
| `ctx.memory` | Typed accumulators: `Number`, `Average`, `MinMax`, `Set`, `Map`, `Buffer`, `Boolean`, `BitMask`, `Trigger`. |
|
|
84
|
+
|
|
85
|
+
See `docs/frameworks.md` in the monorepo for the full reference.
|
|
86
|
+
|
|
87
|
+
## Scaffolding
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
dpe-dev scaffold --name my-tool --runtime python --out ./my-tool --description "what it does"
|
|
91
|
+
cd my-tool
|
|
92
|
+
dpe-dev build . # creates .venv, installs the tool with [dev] extras
|
|
93
|
+
dpe-dev test . # pytest inside the venv
|
|
94
|
+
dpe-dev verify . # spawn the tool's main.py, feed input.ndjson, diff stdout vs expected.ndjson
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Optional speed-up
|
|
98
|
+
|
|
99
|
+
For bulk-hashing payloads, install `xxhash` and pass `algorithm="xxhash"`
|
|
100
|
+
to `dpe.hash_file()`. The framework falls back to `hashlib.blake2b` when
|
|
101
|
+
xxhash is missing.
|
|
102
|
+
|
|
103
|
+
## Repo & licence
|
|
104
|
+
|
|
105
|
+
- Source: <https://github.com/combycode/dpe-framework-python>
|
|
106
|
+
- Monorepo (canonical): <https://github.com/combycode/dpe>
|
|
107
|
+
- Licence: AGPL-3.0-or-later
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "combycode-dpe"
|
|
7
|
+
version = "2.0.0rc1"
|
|
8
|
+
description = "DPE framework for building streaming pipeline tools in Python"
|
|
9
|
+
license = "AGPL-3.0-or-later"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = []
|
|
12
|
+
|
|
13
|
+
[project.optional-dependencies]
|
|
14
|
+
dev = ["pytest>=8.0", "jsonschema>=4.20", "mypy>=1.10"]
|
|
15
|
+
|
|
16
|
+
[project.urls]
|
|
17
|
+
Repository = "https://github.com/combycode/dpe-framework-python"
|
|
18
|
+
|
|
19
|
+
[tool.hatch.build.targets.wheel]
|
|
20
|
+
packages = ["src/dpe"]
|
|
21
|
+
|
|
22
|
+
[tool.ruff]
|
|
23
|
+
line-length = 100
|
|
24
|
+
target-version = "py311"
|
|
25
|
+
extend-exclude = ["template"]
|
|
26
|
+
|
|
27
|
+
[tool.ruff.lint]
|
|
28
|
+
select = [
|
|
29
|
+
"E", # pycodestyle errors
|
|
30
|
+
"F", # pyflakes
|
|
31
|
+
"W", # pycodestyle warnings
|
|
32
|
+
"I", # isort
|
|
33
|
+
"B", # flake8-bugbear
|
|
34
|
+
"UP", # pyupgrade
|
|
35
|
+
]
|
|
36
|
+
ignore = []
|
|
37
|
+
|
|
38
|
+
[tool.ruff.format]
|
|
39
|
+
quote-style = "double"
|
|
40
|
+
indent-style = "space"
|
|
41
|
+
|
|
42
|
+
[tool.mypy]
|
|
43
|
+
python_version = "3.11"
|
|
44
|
+
files = ["src/dpe"]
|
|
45
|
+
# Pragmatic strictness: catch real type bugs without forcing every helper
|
|
46
|
+
# to carry full annotations. Bump to `strict = true` in a follow-up pass
|
|
47
|
+
# when the codebase is fully annotated.
|
|
48
|
+
no_implicit_optional = true
|
|
49
|
+
warn_redundant_casts = true
|
|
50
|
+
warn_unused_ignores = true
|
|
51
|
+
check_untyped_defs = false
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DPE — Data Processing Engine framework.
|
|
3
|
+
|
|
4
|
+
Build streaming pipeline tools with minimal boilerplate.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
import dpe
|
|
8
|
+
|
|
9
|
+
def process_input(v, settings, ctx):
|
|
10
|
+
ctx.output({"result": transform(v)})
|
|
11
|
+
ctx.emit("validate", {"data": v})
|
|
12
|
+
|
|
13
|
+
def process_validate(v, settings, ctx):
|
|
14
|
+
if ok(v):
|
|
15
|
+
ctx.output(v)
|
|
16
|
+
else:
|
|
17
|
+
ctx.error(v, "validation failed")
|
|
18
|
+
|
|
19
|
+
if __name__ == "__main__":
|
|
20
|
+
dpe.run()
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import inspect
|
|
24
|
+
|
|
25
|
+
from dpe._accumulators import (
|
|
26
|
+
Average,
|
|
27
|
+
BitMask,
|
|
28
|
+
Boolean,
|
|
29
|
+
Buffer,
|
|
30
|
+
Map,
|
|
31
|
+
Memory,
|
|
32
|
+
MinMax,
|
|
33
|
+
Number,
|
|
34
|
+
Set,
|
|
35
|
+
Trigger,
|
|
36
|
+
)
|
|
37
|
+
from dpe._context import Context
|
|
38
|
+
from dpe._envelope import hash_file, hash_string
|
|
39
|
+
from dpe._runtime import Runtime
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def run():
|
|
43
|
+
"""Start the tool processing loop.
|
|
44
|
+
|
|
45
|
+
Discovers process_input and process_<queue> functions in the calling module,
|
|
46
|
+
parses settings from argv[1], reads stdin line by line, and dispatches.
|
|
47
|
+
|
|
48
|
+
Call this at the bottom of your tool:
|
|
49
|
+
if __name__ == "__main__":
|
|
50
|
+
dpe.run()
|
|
51
|
+
"""
|
|
52
|
+
frame = inspect.stack()[1]
|
|
53
|
+
caller_module = inspect.getmodule(frame[0])
|
|
54
|
+
if caller_module is None:
|
|
55
|
+
raise RuntimeError("Cannot discover caller module")
|
|
56
|
+
|
|
57
|
+
runtime = Runtime()
|
|
58
|
+
runtime.run(caller_module)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
__all__ = [
|
|
62
|
+
"run",
|
|
63
|
+
"hash_string",
|
|
64
|
+
"hash_file",
|
|
65
|
+
"Context",
|
|
66
|
+
"Memory",
|
|
67
|
+
"Number",
|
|
68
|
+
"Average",
|
|
69
|
+
"MinMax",
|
|
70
|
+
"Set",
|
|
71
|
+
"Map",
|
|
72
|
+
"Buffer",
|
|
73
|
+
"Boolean",
|
|
74
|
+
"BitMask",
|
|
75
|
+
"Trigger",
|
|
76
|
+
]
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Typed accumulators for shared memory across processors."""
|
|
2
|
+
|
|
3
|
+
from collections import deque
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Number:
|
|
7
|
+
"""Numeric accumulator with increment/decrement."""
|
|
8
|
+
|
|
9
|
+
__slots__ = ("_value",)
|
|
10
|
+
|
|
11
|
+
def __init__(self, initial: float = 0):
|
|
12
|
+
self._value = initial
|
|
13
|
+
|
|
14
|
+
def inc(self, delta: float = 1) -> float:
|
|
15
|
+
self._value += delta
|
|
16
|
+
return self._value
|
|
17
|
+
|
|
18
|
+
def dec(self, delta: float = 1) -> float:
|
|
19
|
+
self._value -= delta
|
|
20
|
+
return self._value
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def value(self) -> float:
|
|
24
|
+
return self._value
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Average:
|
|
28
|
+
"""Running average without storing all values."""
|
|
29
|
+
|
|
30
|
+
__slots__ = ("_sum", "_count")
|
|
31
|
+
|
|
32
|
+
def __init__(self):
|
|
33
|
+
self._sum = 0.0
|
|
34
|
+
self._count = 0
|
|
35
|
+
|
|
36
|
+
def add(self, value: float):
|
|
37
|
+
self._sum += value
|
|
38
|
+
self._count += 1
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def value(self) -> float:
|
|
42
|
+
if self._count == 0:
|
|
43
|
+
return 0.0
|
|
44
|
+
return self._sum / self._count
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def count(self) -> int:
|
|
48
|
+
return self._count
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class MinMax:
|
|
52
|
+
"""Track minimum and maximum values."""
|
|
53
|
+
|
|
54
|
+
__slots__ = ("_min", "_max", "_count")
|
|
55
|
+
|
|
56
|
+
def __init__(self):
|
|
57
|
+
self._min = None
|
|
58
|
+
self._max = None
|
|
59
|
+
self._count = 0
|
|
60
|
+
|
|
61
|
+
def add(self, value: float):
|
|
62
|
+
if self._min is None or value < self._min:
|
|
63
|
+
self._min = value
|
|
64
|
+
if self._max is None or value > self._max:
|
|
65
|
+
self._max = value
|
|
66
|
+
self._count += 1
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def min(self) -> float | None:
|
|
70
|
+
return self._min
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def max(self) -> float | None:
|
|
74
|
+
return self._max
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def count(self) -> int:
|
|
78
|
+
return self._count
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Set:
|
|
82
|
+
"""Unique value collection."""
|
|
83
|
+
|
|
84
|
+
__slots__ = ("_data",)
|
|
85
|
+
|
|
86
|
+
def __init__(self):
|
|
87
|
+
self._data = set()
|
|
88
|
+
|
|
89
|
+
def add(self, value):
|
|
90
|
+
self._data.add(value)
|
|
91
|
+
|
|
92
|
+
def has(self, value) -> bool:
|
|
93
|
+
return value in self._data
|
|
94
|
+
|
|
95
|
+
def delete(self, value):
|
|
96
|
+
self._data.discard(value)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def size(self) -> int:
|
|
100
|
+
return len(self._data)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def values(self) -> list:
|
|
104
|
+
return list(self._data)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class Map:
|
|
108
|
+
"""Key-value lookup. Transient storage, not for accumulation."""
|
|
109
|
+
|
|
110
|
+
__slots__ = ("_data",)
|
|
111
|
+
|
|
112
|
+
def __init__(self):
|
|
113
|
+
self._data = {}
|
|
114
|
+
|
|
115
|
+
def set(self, key: str, value):
|
|
116
|
+
self._data[key] = value
|
|
117
|
+
|
|
118
|
+
def get(self, key: str, default=None):
|
|
119
|
+
return self._data.get(key, default)
|
|
120
|
+
|
|
121
|
+
def has(self, key: str) -> bool:
|
|
122
|
+
return key in self._data
|
|
123
|
+
|
|
124
|
+
def delete(self, key: str):
|
|
125
|
+
self._data.pop(key, None)
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def keys(self) -> list:
|
|
129
|
+
return list(self._data.keys())
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def size(self) -> int:
|
|
133
|
+
return len(self._data)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class Buffer:
|
|
137
|
+
"""Ordered collection with optional size limit (sliding window)."""
|
|
138
|
+
|
|
139
|
+
__slots__ = ("_data", "_limit")
|
|
140
|
+
|
|
141
|
+
def __init__(self, limit: int | None = None):
|
|
142
|
+
self._data: deque = deque(maxlen=limit)
|
|
143
|
+
self._limit = limit
|
|
144
|
+
|
|
145
|
+
def push(self, item):
|
|
146
|
+
self._data.append(item)
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def items(self) -> list:
|
|
150
|
+
return list(self._data)
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def size(self) -> int:
|
|
154
|
+
return len(self._data)
|
|
155
|
+
|
|
156
|
+
def clear(self):
|
|
157
|
+
self._data.clear()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class Boolean:
|
|
161
|
+
"""Simple flag."""
|
|
162
|
+
|
|
163
|
+
__slots__ = ("_value",)
|
|
164
|
+
|
|
165
|
+
def __init__(self, initial: bool = False):
|
|
166
|
+
self._value = initial
|
|
167
|
+
|
|
168
|
+
def set(self, value: bool = True):
|
|
169
|
+
self._value = value
|
|
170
|
+
|
|
171
|
+
def toggle(self, value: bool | None = None):
|
|
172
|
+
if value is not None:
|
|
173
|
+
self._value = value
|
|
174
|
+
else:
|
|
175
|
+
self._value = not self._value
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def value(self) -> bool:
|
|
179
|
+
return self._value
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class BitMask:
|
|
183
|
+
"""Named flag set for tracking multiple boolean states."""
|
|
184
|
+
|
|
185
|
+
__slots__ = ("_flags",)
|
|
186
|
+
|
|
187
|
+
def __init__(self, names: list[str]):
|
|
188
|
+
self._flags = {name: False for name in names}
|
|
189
|
+
|
|
190
|
+
def set(self, name: str):
|
|
191
|
+
if name not in self._flags:
|
|
192
|
+
raise KeyError(f"Unknown flag: {name}")
|
|
193
|
+
self._flags[name] = True
|
|
194
|
+
|
|
195
|
+
def get(self, name: str) -> bool:
|
|
196
|
+
if name not in self._flags:
|
|
197
|
+
raise KeyError(f"Unknown flag: {name}")
|
|
198
|
+
return self._flags[name]
|
|
199
|
+
|
|
200
|
+
def clear(self, name: str):
|
|
201
|
+
if name not in self._flags:
|
|
202
|
+
raise KeyError(f"Unknown flag: {name}")
|
|
203
|
+
self._flags[name] = False
|
|
204
|
+
|
|
205
|
+
def all(self) -> bool:
|
|
206
|
+
return all(self._flags.values())
|
|
207
|
+
|
|
208
|
+
def any(self) -> bool:
|
|
209
|
+
return any(self._flags.values())
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class Trigger:
|
|
213
|
+
"""Condition checker across accumulators. Condition is a callable(memory) → bool."""
|
|
214
|
+
|
|
215
|
+
__slots__ = ("_condition", "_memory_ref")
|
|
216
|
+
|
|
217
|
+
def __init__(self, condition, memory):
|
|
218
|
+
self._condition = condition
|
|
219
|
+
self._memory_ref = memory
|
|
220
|
+
|
|
221
|
+
def check(self) -> bool:
|
|
222
|
+
return self._condition(self._memory_ref)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class Memory:
|
|
226
|
+
"""Shared memory — typed accumulator registry.
|
|
227
|
+
|
|
228
|
+
Accumulators are created on first access by name.
|
|
229
|
+
Same name always returns the same instance.
|
|
230
|
+
"""
|
|
231
|
+
|
|
232
|
+
def __init__(self):
|
|
233
|
+
self._store: dict = {}
|
|
234
|
+
|
|
235
|
+
def number(self, name: str, initial: float = 0) -> Number:
|
|
236
|
+
return self._get_or_create(name, Number, initial)
|
|
237
|
+
|
|
238
|
+
def average(self, name: str) -> Average:
|
|
239
|
+
return self._get_or_create(name, Average)
|
|
240
|
+
|
|
241
|
+
def minmax(self, name: str) -> MinMax:
|
|
242
|
+
return self._get_or_create(name, MinMax)
|
|
243
|
+
|
|
244
|
+
def set(self, name: str) -> Set:
|
|
245
|
+
return self._get_or_create(name, Set)
|
|
246
|
+
|
|
247
|
+
def map(self, name: str) -> Map:
|
|
248
|
+
return self._get_or_create(name, Map)
|
|
249
|
+
|
|
250
|
+
def buffer(self, name: str, limit: int | None = None) -> Buffer:
|
|
251
|
+
return self._get_or_create(name, Buffer, limit)
|
|
252
|
+
|
|
253
|
+
def boolean(self, name: str, initial: bool = False) -> Boolean:
|
|
254
|
+
return self._get_or_create(name, Boolean, initial)
|
|
255
|
+
|
|
256
|
+
def bitmask(self, name: str, flags: list[str]) -> BitMask:
|
|
257
|
+
return self._get_or_create(name, BitMask, flags)
|
|
258
|
+
|
|
259
|
+
def trigger(self, name: str, condition=None) -> Trigger:
|
|
260
|
+
if name in self._store:
|
|
261
|
+
return self._store[name]
|
|
262
|
+
if condition is None:
|
|
263
|
+
raise ValueError(f"Trigger '{name}' does not exist and no condition provided")
|
|
264
|
+
t = Trigger(condition, self)
|
|
265
|
+
self._store[name] = t
|
|
266
|
+
return t
|
|
267
|
+
|
|
268
|
+
def get(self, name: str):
|
|
269
|
+
"""Get accumulator by name. Returns None if not found."""
|
|
270
|
+
return self._store.get(name)
|
|
271
|
+
|
|
272
|
+
def delete(self, name: str):
|
|
273
|
+
"""Remove accumulator by name."""
|
|
274
|
+
self._store.pop(name, None)
|
|
275
|
+
|
|
276
|
+
def has(self, name: str) -> bool:
|
|
277
|
+
return name in self._store
|
|
278
|
+
|
|
279
|
+
def _get_or_create(self, name, cls, *args):
|
|
280
|
+
existing = self._store.get(name)
|
|
281
|
+
if existing is not None:
|
|
282
|
+
if not isinstance(existing, cls):
|
|
283
|
+
raise TypeError(
|
|
284
|
+
f"Accumulator '{name}' exists as {type(existing).__name__}, "
|
|
285
|
+
f"not {cls.__name__}"
|
|
286
|
+
)
|
|
287
|
+
return existing
|
|
288
|
+
instance = cls(*args)
|
|
289
|
+
self._store[name] = instance
|
|
290
|
+
return instance
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Context object — passed to every processor invocation."""
|
|
2
|
+
|
|
3
|
+
from dpe._accumulators import Memory
|
|
4
|
+
from dpe._envelope import (
|
|
5
|
+
hash_file,
|
|
6
|
+
hash_string,
|
|
7
|
+
write_data,
|
|
8
|
+
write_error,
|
|
9
|
+
write_log,
|
|
10
|
+
write_meta,
|
|
11
|
+
write_stats,
|
|
12
|
+
write_trace,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Context:
|
|
17
|
+
"""Processing context created per invocation.
|
|
18
|
+
|
|
19
|
+
Provides framework API: output, emit, drain, meta, log, error, trace, stats, hash.
|
|
20
|
+
Each processor call gets its own ctx with current id/src.
|
|
21
|
+
Memory is shared across all processors in the tool.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
__slots__ = ("id", "src", "memory", "_runtime", "_labels")
|
|
25
|
+
|
|
26
|
+
def __init__(self, id: str, src: str, memory: Memory, runtime):
|
|
27
|
+
self.id = id
|
|
28
|
+
self.src = src
|
|
29
|
+
self.memory = memory
|
|
30
|
+
self._runtime = runtime
|
|
31
|
+
# Labels accumulated by ctx.trace(k, v). Flushed as one merged
|
|
32
|
+
# {type:"trace"} stderr event before each ctx.output(), then cleared.
|
|
33
|
+
self._labels: dict = {}
|
|
34
|
+
|
|
35
|
+
def trace(self, key: str, value):
|
|
36
|
+
"""Attach a label to this invocation's next output envelope.
|
|
37
|
+
|
|
38
|
+
Call as many times as needed between ctx.output() calls. The
|
|
39
|
+
accumulated labels are flushed once (merged) as a {type:"trace"}
|
|
40
|
+
event to stderr just before ctx.output() writes the envelope,
|
|
41
|
+
then cleared.
|
|
42
|
+
"""
|
|
43
|
+
self._labels[key] = value
|
|
44
|
+
|
|
45
|
+
def stats(self, **kwargs):
|
|
46
|
+
"""Emit a stats event to stderr. Example: ctx.stats(rows=1000, rps=250)."""
|
|
47
|
+
if kwargs:
|
|
48
|
+
write_stats(kwargs)
|
|
49
|
+
|
|
50
|
+
def output(self, v, *, id: str | None = None, src: str | None = None):
|
|
51
|
+
"""Emit data record to stdout.
|
|
52
|
+
|
|
53
|
+
Flushes accumulated trace labels as a merged trace event first,
|
|
54
|
+
then writes the envelope, then clears the label bag.
|
|
55
|
+
"""
|
|
56
|
+
out_id = id if id is not None else self.id
|
|
57
|
+
out_src = src if src is not None else self.src
|
|
58
|
+
# Emit trace even with empty labels — the chain row itself is the value.
|
|
59
|
+
write_trace(out_id, out_src, self._labels)
|
|
60
|
+
self._labels = {}
|
|
61
|
+
write_data(v, out_id, out_src)
|
|
62
|
+
|
|
63
|
+
def emit(self, queue: str, v, *, id: str | None = None, src: str | None = None):
|
|
64
|
+
"""Emit to internal named queue. Processed by process_<queue>().
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
queue: Queue name (maps to process_<queue> function)
|
|
68
|
+
v: Payload data
|
|
69
|
+
id: Override id. Defaults to current ctx.id.
|
|
70
|
+
src: Override src. Defaults to current ctx.src.
|
|
71
|
+
"""
|
|
72
|
+
item_id = id if id is not None else self.id
|
|
73
|
+
item_src = src if src is not None else self.src
|
|
74
|
+
self._runtime.enqueue(queue, v, item_id, item_src)
|
|
75
|
+
|
|
76
|
+
def drain(self):
|
|
77
|
+
"""Block until all queued items are processed.
|
|
78
|
+
|
|
79
|
+
After drain returns, all accumulators reflect completed processing.
|
|
80
|
+
Can be called multiple times within one processor call.
|
|
81
|
+
"""
|
|
82
|
+
self._runtime.drain_queue()
|
|
83
|
+
|
|
84
|
+
def meta(self, v: dict):
|
|
85
|
+
"""Emit metadata record to stdout."""
|
|
86
|
+
write_meta(v)
|
|
87
|
+
|
|
88
|
+
def log(self, msg: str, *, level: str = "info", **extra):
|
|
89
|
+
"""Write structured log to stderr."""
|
|
90
|
+
write_log(msg, level, extra if extra else None)
|
|
91
|
+
|
|
92
|
+
def error(self, v, err):
|
|
93
|
+
"""Write error to stderr with original input preserved."""
|
|
94
|
+
write_error(v, err, self.id, self.src)
|
|
95
|
+
|
|
96
|
+
def hash(self, key: str) -> str:
|
|
97
|
+
"""Hash a string. Returns 16-char hex."""
|
|
98
|
+
return hash_string(key)
|
|
99
|
+
|
|
100
|
+
def hash_file(self, filepath: str, algorithm: str = "blake2b") -> str | None:
|
|
101
|
+
"""Hash file content in chunks. Returns hex string or None on error."""
|
|
102
|
+
return hash_file(filepath, algorithm)
|