combycode-dpe 2.0.3__tar.gz → 2.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. combycode_dpe-2.0.4/PKG-INFO +128 -0
  2. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/README.md +4 -2
  3. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/pyproject.toml +7 -2
  4. combycode_dpe-2.0.3/PKG-INFO +0 -11
  5. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/.gitignore +0 -0
  6. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/__init__.py +0 -0
  7. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_accumulators.py +0 -0
  8. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_cache.py +0 -0
  9. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_context.py +0 -0
  10. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_envelope.py +0 -0
  11. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_paths.py +0 -0
  12. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/src/dpe/_runtime.py +0 -0
  13. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/meta.json +0 -0
  14. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/pyproject.toml +0 -0
  15. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/spec.yaml +0 -0
  16. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/src/__PKG__/__init__.py +0 -0
  17. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/src/__PKG__/main.py +0 -0
  18. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/template/tests/test_basic.py +0 -0
  19. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_accumulators.py +0 -0
  20. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_cache.py +0 -0
  21. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_integration.py +0 -0
  22. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_paths.py +0 -0
  23. {combycode_dpe-2.0.3 → combycode_dpe-2.0.4}/tests/test_stderr_events_conformance.py +0 -0
@@ -0,0 +1,128 @@
1
+ Metadata-Version: 2.4
2
+ Name: combycode-dpe
3
+ Version: 2.0.4
4
+ Summary: DPE framework for building streaming pipeline tools in Python
5
+ Project-URL: Homepage, https://github.com/combycode/dpe
6
+ Project-URL: Repository, https://github.com/combycode/dpe
7
+ Project-URL: Documentation, https://github.com/combycode/dpe/blob/main/frameworks/python/README.md
8
+ Project-URL: Issues, https://github.com/combycode/dpe/issues
9
+ Project-URL: Changelog, https://github.com/combycode/dpe/blob/main/CHANGELOG.md
10
+ License-Expression: AGPL-3.0-or-later
11
+ Requires-Python: >=3.11
12
+ Provides-Extra: dev
13
+ Requires-Dist: jsonschema>=4.20; extra == 'dev'
14
+ Requires-Dist: mypy>=1.10; extra == 'dev'
15
+ Requires-Dist: pytest>=8.0; extra == 'dev'
16
+ Description-Content-Type: text/markdown
17
+
18
+ # combycode-dpe (Python)
19
+
20
+ DPE framework for building streaming pipeline tools in Python.
21
+
22
+ A DPE tool is an independent program: it reads NDJSON envelopes from stdin,
23
+ processes each one, and writes NDJSON envelopes to stdout. The runner
24
+ (`dpe`) spawns it, pipes data through it, and routes its output to the
25
+ next stage. The framework handles the loop, the parsing, the queues, and
26
+ the error/trace plumbing — your code only writes the transform.
27
+
28
+ Requires Python 3.11+.
29
+
30
+ ## Install
31
+
32
+ ```bash
33
+ pip install combycode-dpe
34
+ # or in a per-tool venv:
35
+ uv venv --seed .venv && .venv/bin/pip install combycode-dpe
36
+ ```
37
+
38
+ ## Hello tool
39
+
40
+ ```python
41
+ # src/my_tool/main.py
42
+ import dpe
43
+
44
+ def process_input(v, settings, ctx):
45
+ ctx.output(v) # pass-through
46
+
47
+ if __name__ == "__main__":
48
+ dpe.run()
49
+ ```
50
+
51
+ Run as a tool: `python src/my_tool/main.py '{"...settings JSON..."}'`. The
52
+ runner passes settings as `argv[1]` and pipes NDJSON through stdin/stdout.
53
+
54
+ `dpe.run()` introspects the calling module to find `process_input` and
55
+ any `process_<queue>` handlers — no manual registration needed.
56
+
57
+ ## Processor signature
58
+
59
+ ```python
60
+ def process_input(v, settings, ctx):
61
+ ...
62
+ ```
63
+
64
+ - `v` — envelope payload (a dict / list / scalar — whatever JSON parsed to).
65
+ - `settings` — argv[1] parsed once at startup.
66
+ - `ctx` — emits output, errors, logs, metadata; reads/writes shared
67
+ accumulators in `ctx.memory`.
68
+
69
+ ## Internal queues
70
+
71
+ ```python
72
+ import dpe
73
+
74
+ def process_input(v, settings, ctx):
75
+ ctx.emit("validate", v)
76
+
77
+ def process_validate(v, settings, ctx):
78
+ if v.get("ok"):
79
+ ctx.output(v)
80
+ else:
81
+ ctx.error(v, "validation failed")
82
+
83
+ if __name__ == "__main__":
84
+ dpe.run()
85
+ ```
86
+
87
+ `ctx.emit(queue, v)` enqueues for the named handler. `ctx.drain()` blocks
88
+ until every queue is empty.
89
+
90
+ ## Context API (essentials)
91
+
92
+ | Call | What it does |
93
+ |---|---|
94
+ | `ctx.output(v, id=None, src=None)` | Emit `{t:"d",...}` to stdout. `id`/`src` default to inherit. |
95
+ | `ctx.emit(queue, v)` | Push to internal queue, processed by `process_<queue>`. |
96
+ | `ctx.drain()` | Block until all queues are empty. |
97
+ | `ctx.error(v, msg)` | Write `{type:"error",input:v,error:msg,...}` to stderr. |
98
+ | `ctx.log(level, msg)` | Structured stderr log. |
99
+ | `ctx.meta(v)` | Emit `{t:"m", v}` envelope to stdout. |
100
+ | `ctx.memory` | Typed accumulators: `Number`, `Average`, `MinMax`, `Set`, `Map`, `Buffer`, `Boolean`, `BitMask`, `Trigger`. |
101
+
102
+ See `docs/frameworks.md` in the monorepo for the full reference.
103
+
104
+ ## Scaffolding
105
+
106
+ ```bash
107
+ dpe-dev scaffold --name my-tool --runtime python --out ./my-tool --description "what it does"
108
+ cd my-tool
109
+ dpe-dev build . # creates .venv, installs the tool with [dev] extras
110
+ dpe-dev test . # pytest inside the venv
111
+ ```
112
+
113
+ End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
114
+ tool is wired into a pipeline — see `docs/testing.md`.
115
+
116
+ ## Optional speed-up
117
+
118
+ For bulk-hashing payloads, install `xxhash` and pass `algorithm="xxhash"`
119
+ to `dpe.hash_file()`. The framework falls back to `hashlib.blake2b` when
120
+ xxhash is missing.
121
+
122
+ ## Repo & licence
123
+
124
+ - Source: <https://github.com/combycode/dpe/tree/main/frameworks/python>
125
+ - Monorepo: <https://github.com/combycode/dpe>
126
+ - Issues: <https://github.com/combycode/dpe/issues>
127
+ - Changelog: <https://github.com/combycode/dpe/blob/main/CHANGELOG.md>
128
+ - Licence: AGPL-3.0-or-later
@@ -104,6 +104,8 @@ xxhash is missing.
104
104
 
105
105
  ## Repo & licence
106
106
 
107
- - Source: <https://github.com/combycode/dpe-framework-python>
108
- - Monorepo (canonical): <https://github.com/combycode/dpe>
107
+ - Source: <https://github.com/combycode/dpe/tree/main/frameworks/python>
108
+ - Monorepo: <https://github.com/combycode/dpe>
109
+ - Issues: <https://github.com/combycode/dpe/issues>
110
+ - Changelog: <https://github.com/combycode/dpe/blob/main/CHANGELOG.md>
109
111
  - Licence: AGPL-3.0-or-later
@@ -4,8 +4,9 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "combycode-dpe"
7
- version = "2.0.3"
7
+ version = "2.0.4"
8
8
  description = "DPE framework for building streaming pipeline tools in Python"
9
+ readme = "README.md"
9
10
  license = "AGPL-3.0-or-later"
10
11
  requires-python = ">=3.11"
11
12
  dependencies = []
@@ -14,7 +15,11 @@ dependencies = []
14
15
  dev = ["pytest>=8.0", "jsonschema>=4.20", "mypy>=1.10"]
15
16
 
16
17
  [project.urls]
17
- Repository = "https://github.com/combycode/dpe-framework-python"
18
+ Homepage = "https://github.com/combycode/dpe"
19
+ Repository = "https://github.com/combycode/dpe"
20
+ Documentation = "https://github.com/combycode/dpe/blob/main/frameworks/python/README.md"
21
+ Issues = "https://github.com/combycode/dpe/issues"
22
+ Changelog = "https://github.com/combycode/dpe/blob/main/CHANGELOG.md"
18
23
 
19
24
  [tool.hatch.build.targets.wheel]
20
25
  packages = ["src/dpe"]
@@ -1,11 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: combycode-dpe
3
- Version: 2.0.3
4
- Summary: DPE framework for building streaming pipeline tools in Python
5
- Project-URL: Repository, https://github.com/combycode/dpe-framework-python
6
- License-Expression: AGPL-3.0-or-later
7
- Requires-Python: >=3.11
8
- Provides-Extra: dev
9
- Requires-Dist: jsonschema>=4.20; extra == 'dev'
10
- Requires-Dist: mypy>=1.10; extra == 'dev'
11
- Requires-Dist: pytest>=8.0; extra == 'dev'
File without changes