combycode-dpe 2.0.3__tar.gz → 2.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- combycode_dpe-2.0.4/PKG-INFO +128 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/README.md +4 -2
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/pyproject.toml +7 -2
- combycode_dpe-2.0.3/PKG-INFO +0 -11
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/.gitignore +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/__init__.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_accumulators.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_cache.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_context.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_envelope.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_paths.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_runtime.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/meta.json +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/pyproject.toml +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/spec.yaml +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/src/__PKG__/__init__.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/src/__PKG__/main.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/tests/test_basic.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_accumulators.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_cache.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_integration.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_paths.py +0 -0
- {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_stderr_events_conformance.py +0 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: combycode-dpe
|
|
3
|
+
Version: 2.0.4
|
|
4
|
+
Summary: DPE framework for building streaming pipeline tools in Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/combycode/dpe
|
|
6
|
+
Project-URL: Repository, https://github.com/combycode/dpe
|
|
7
|
+
Project-URL: Documentation, https://github.com/combycode/dpe/blob/main/frameworks/python/README.md
|
|
8
|
+
Project-URL: Issues, https://github.com/combycode/dpe/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/combycode/dpe/blob/main/CHANGELOG.md
|
|
10
|
+
License-Expression: AGPL-3.0-or-later
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: jsonschema>=4.20; extra == 'dev'
|
|
14
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
15
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# combycode-dpe (Python)
|
|
19
|
+
|
|
20
|
+
DPE framework for building streaming pipeline tools in Python.
|
|
21
|
+
|
|
22
|
+
A DPE tool is an independent program: it reads NDJSON envelopes from stdin,
|
|
23
|
+
processes each one, and writes NDJSON envelopes to stdout. The runner
|
|
24
|
+
(`dpe`) spawns it, pipes data through it, and routes its output to the
|
|
25
|
+
next stage. The framework handles the loop, the parsing, the queues, and
|
|
26
|
+
the error/trace plumbing — your code only writes the transform.
|
|
27
|
+
|
|
28
|
+
Requires Python 3.11+.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install combycode-dpe
|
|
34
|
+
# or in a per-tool venv:
|
|
35
|
+
uv venv --seed .venv && .venv/bin/pip install combycode-dpe
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Hello tool
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
# src/my_tool/main.py
|
|
42
|
+
import dpe
|
|
43
|
+
|
|
44
|
+
def process_input(v, settings, ctx):
|
|
45
|
+
ctx.output(v) # pass-through
|
|
46
|
+
|
|
47
|
+
if __name__ == "__main__":
|
|
48
|
+
dpe.run()
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Run as a tool: `python src/my_tool/main.py '{"...settings JSON..."}'`. The
|
|
52
|
+
runner passes settings as `argv[1]` and pipes NDJSON through stdin/stdout.
|
|
53
|
+
|
|
54
|
+
`dpe.run()` introspects the calling module to find `process_input` and
|
|
55
|
+
any `process_<queue>` handlers — no manual registration needed.
|
|
56
|
+
|
|
57
|
+
## Processor signature
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
def process_input(v, settings, ctx):
|
|
61
|
+
...
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
- `v` — envelope payload (a dict / list / scalar — whatever JSON parsed to).
|
|
65
|
+
- `settings` — argv[1] parsed once at startup.
|
|
66
|
+
- `ctx` — emits output, errors, logs, metadata; reads/writes shared
|
|
67
|
+
accumulators in `ctx.memory`.
|
|
68
|
+
|
|
69
|
+
## Internal queues
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
import dpe
|
|
73
|
+
|
|
74
|
+
def process_input(v, settings, ctx):
|
|
75
|
+
ctx.emit("validate", v)
|
|
76
|
+
|
|
77
|
+
def process_validate(v, settings, ctx):
|
|
78
|
+
if v.get("ok"):
|
|
79
|
+
ctx.output(v)
|
|
80
|
+
else:
|
|
81
|
+
ctx.error(v, "validation failed")
|
|
82
|
+
|
|
83
|
+
if __name__ == "__main__":
|
|
84
|
+
dpe.run()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`ctx.emit(queue, v)` enqueues for the named handler. `ctx.drain()` blocks
|
|
88
|
+
until every queue is empty.
|
|
89
|
+
|
|
90
|
+
## Context API (essentials)
|
|
91
|
+
|
|
92
|
+
| Call | What it does |
|
|
93
|
+
|---|---|
|
|
94
|
+
| `ctx.output(v, id=None, src=None)` | Emit `{t:"d",...}` to stdout. `id`/`src` default to inherit. |
|
|
95
|
+
| `ctx.emit(queue, v)` | Push to internal queue, processed by `process_<queue>`. |
|
|
96
|
+
| `ctx.drain()` | Block until all queues are empty. |
|
|
97
|
+
| `ctx.error(v, msg)` | Write `{type:"error",input:v,error:msg,...}` to stderr. |
|
|
98
|
+
| `ctx.log(level, msg)` | Structured stderr log. |
|
|
99
|
+
| `ctx.meta(v)` | Emit `{t:"m", v}` envelope to stdout. |
|
|
100
|
+
| `ctx.memory` | Typed accumulators: `Number`, `Average`, `MinMax`, `Set`, `Map`, `Buffer`, `Boolean`, `BitMask`, `Trigger`. |
|
|
101
|
+
|
|
102
|
+
See `docs/frameworks.md` in the monorepo for the full reference.
|
|
103
|
+
|
|
104
|
+
## Scaffolding
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
dpe-dev scaffold --name my-tool --runtime python --out ./my-tool --description "what it does"
|
|
108
|
+
cd my-tool
|
|
109
|
+
dpe-dev build . # creates .venv, installs the tool with [dev] extras
|
|
110
|
+
dpe-dev test . # pytest inside the venv
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
|
|
114
|
+
tool is wired into a pipeline — see `docs/testing.md`.
|
|
115
|
+
|
|
116
|
+
## Optional speed-up
|
|
117
|
+
|
|
118
|
+
For bulk-hashing payloads, install `xxhash` and pass `algorithm="xxhash"`
|
|
119
|
+
to `dpe.hash_file()`. The framework falls back to `hashlib.blake2b` when
|
|
120
|
+
xxhash is missing.
|
|
121
|
+
|
|
122
|
+
## Repo & licence
|
|
123
|
+
|
|
124
|
+
- Source: <https://github.com/combycode/dpe/tree/main/frameworks/python>
|
|
125
|
+
- Monorepo: <https://github.com/combycode/dpe>
|
|
126
|
+
- Issues: <https://github.com/combycode/dpe/issues>
|
|
127
|
+
- Changelog: <https://github.com/combycode/dpe/blob/main/CHANGELOG.md>
|
|
128
|
+
- Licence: AGPL-3.0-or-later
|
|
@@ -104,6 +104,8 @@ xxhash is missing.
|
|
|
104
104
|
|
|
105
105
|
## Repo & licence
|
|
106
106
|
|
|
107
|
-
- Source: <https://github.com/combycode/dpe
|
|
108
|
-
- Monorepo
|
|
107
|
+
- Source: <https://github.com/combycode/dpe/tree/main/frameworks/python>
|
|
108
|
+
- Monorepo: <https://github.com/combycode/dpe>
|
|
109
|
+
- Issues: <https://github.com/combycode/dpe/issues>
|
|
110
|
+
- Changelog: <https://github.com/combycode/dpe/blob/main/CHANGELOG.md>
|
|
109
111
|
- Licence: AGPL-3.0-or-later
|
|
@@ -4,8 +4,9 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "combycode-dpe"
|
|
7
|
-
version = "2.0.
|
|
7
|
+
version = "2.0.4"
|
|
8
8
|
description = "DPE framework for building streaming pipeline tools in Python"
|
|
9
|
+
readme = "README.md"
|
|
9
10
|
license = "AGPL-3.0-or-later"
|
|
10
11
|
requires-python = ">=3.11"
|
|
11
12
|
dependencies = []
|
|
@@ -14,7 +15,11 @@ dependencies = []
|
|
|
14
15
|
dev = ["pytest>=8.0", "jsonschema>=4.20", "mypy>=1.10"]
|
|
15
16
|
|
|
16
17
|
[project.urls]
|
|
17
|
-
|
|
18
|
+
Homepage = "https://github.com/combycode/dpe"
|
|
19
|
+
Repository = "https://github.com/combycode/dpe"
|
|
20
|
+
Documentation = "https://github.com/combycode/dpe/blob/main/frameworks/python/README.md"
|
|
21
|
+
Issues = "https://github.com/combycode/dpe/issues"
|
|
22
|
+
Changelog = "https://github.com/combycode/dpe/blob/main/CHANGELOG.md"
|
|
18
23
|
|
|
19
24
|
[tool.hatch.build.targets.wheel]
|
|
20
25
|
packages = ["src/dpe"]
|
combycode_dpe-2.0.3/PKG-INFO
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: combycode-dpe
|
|
3
|
-
Version: 2.0.3
|
|
4
|
-
Summary: DPE framework for building streaming pipeline tools in Python
|
|
5
|
-
Project-URL: Repository, https://github.com/combycode/dpe-framework-python
|
|
6
|
-
License-Expression: AGPL-3.0-or-later
|
|
7
|
-
Requires-Python: >=3.11
|
|
8
|
-
Provides-Extra: dev
|
|
9
|
-
Requires-Dist: jsonschema>=4.20; extra == 'dev'
|
|
10
|
-
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
11
|
-
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|