hexa-pipelines 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hexa_pipelines-0.2.0/LICENSE.md +27 -0
- hexa_pipelines-0.2.0/PKG-INFO +7 -0
- hexa_pipelines-0.2.0/README.md +155 -0
- hexa_pipelines-0.2.0/pyproject.toml +32 -0
- hexa_pipelines-0.2.0/setup.cfg +4 -0
- hexa_pipelines-0.2.0/src/py/hexa/__init__.py +7 -0
- hexa_pipelines-0.2.0/src/py/hexa/codec.py +88 -0
- hexa_pipelines-0.2.0/src/py/hexa/combinators.py +466 -0
- hexa_pipelines-0.2.0/src/py/hexa/core.py +405 -0
- hexa_pipelines-0.2.0/src/py/hexa/encoding.py +307 -0
- hexa_pipelines-0.2.0/src/py/hexa/instructions.py +123 -0
- hexa_pipelines-0.2.0/src/py/hexa/profile.py +280 -0
- hexa_pipelines-0.2.0/src/py/hexa/runtime.py +648 -0
- hexa_pipelines-0.2.0/src/py/hexa/selection_request.py +187 -0
- hexa_pipelines-0.2.0/src/py/hexa/selector.py +77 -0
- hexa_pipelines-0.2.0/src/py/hexa/shape.py +872 -0
- hexa_pipelines-0.2.0/src/py/hexa/shapes.py +40 -0
- hexa_pipelines-0.2.0/src/py/hexa/stage.py +114 -0
- hexa_pipelines-0.2.0/src/py/hexa/template.py +235 -0
- hexa_pipelines-0.2.0/src/py/hexa/vocabulary.py +142 -0
- hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/PKG-INFO +7 -0
- hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/SOURCES.txt +29 -0
- hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/dependency_links.txt +1 -0
- hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/top_level.txt +1 -0
- hexa_pipelines-0.2.0/tests/test_combinators.py +241 -0
- hexa_pipelines-0.2.0/tests/test_config_review_regressions.py +73 -0
- hexa_pipelines-0.2.0/tests/test_conformance.py +84 -0
- hexa_pipelines-0.2.0/tests/test_profiles.py +215 -0
- hexa_pipelines-0.2.0/tests/test_runtime.py +601 -0
- hexa_pipelines-0.2.0/tests/test_shapes.py +299 -0
- hexa_pipelines-0.2.0/tests/test_structure_parity.py +133 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Revised BSD License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Sébastien Pierre. All rights reserved.
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without modification,
|
|
6
|
+
are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer. Redistributions in binary
|
|
10
|
+
form must reproduce the above copyright notice, this list of conditions and
|
|
11
|
+
the following disclaimer in the documentation and/or other materials
|
|
12
|
+
provided with the distribution. Neither the name of Sébastien Pierre nor
|
|
13
|
+
the names of its contributors may be used to endorse or
|
|
14
|
+
promote products derived from this software without specific prior written
|
|
15
|
+
permission.
|
|
16
|
+
|
|
17
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
18
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
19
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
20
|
+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
21
|
+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
22
|
+
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
23
|
+
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
24
|
+
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
25
|
+
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
26
|
+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
27
|
+
POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
```
|
|
2
|
+
▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄
|
|
3
|
+
█ ░▓░ ▓ █ ▄▀░▓░ ▓ █ ░▓░ ▓ █ ▄▀░▓░ ▀▄
|
|
4
|
+
█ ▒ ▒ █ ▐▌ ▒ ▒ ▐▌ ▒ ▒ ▐▌ ▐▌ ▒ ▐▌
|
|
5
|
+
█ ░ ░ █ █ ░ ▄▄▄▄▄▄░ █ ░ ░ █ ▒ ░ █ ▓
|
|
6
|
+
█ ▄▓ █ █ █ ▀▄ ▓ ▄▀ ▒ ▒
|
|
7
|
+
▓ █ ▓ █ ▄▀ ▀▄ ░░ ░ ░
|
|
8
|
+
▒ █▀ ▓ ▓ ▀▀▀▀▀▀▀ ▐▌ █▀ ▐▌ ▒▒ ▒ ░░
|
|
9
|
+
▓░ ▓ ▒ ▒░ ▓ ▓░ ▓ ▓ ▒▓░ ▓ ░▒░
|
|
10
|
+
▓▓▒░ ▓ █ ▐▓▒░ ▒ ▓▓▒░ ▓ ▒ ▓▓▒░ ░█ ░▒▓▒
|
|
11
|
+
██▓▒░ ▒ █ ▀▓▒░ ░ ██▓▒░ ▒ ░ ██▓▒░ ▒█ ░▒▓█▓
|
|
12
|
+
▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
*hexa* is a library for writing data pipelines as projections between
|
|
16
|
+
shapes. You declare what to match in the source data (a *shape*, the regex of
|
|
17
|
+
data structures), what to build from it (a *template*), and compose the two
|
|
18
|
+
into stages that select, project, recover and retry, with I/O yielded as
|
|
19
|
+
inert effects that the runtime performs, records and replays.
|
|
20
|
+
|
|
21
|
+
It comes from extraction work: pulling records out of CRM and partner APIs,
|
|
22
|
+
normalising their formats, and reporting what did not fit instead of failing
|
|
23
|
+
or silently guessing. In our view most of such a program is configuration,
|
|
24
|
+
so *hexa* makes the shapes, codecs and pipeline declaration the program, and
|
|
25
|
+
hand-written code the exception. It is a specification first: the Python and
|
|
26
|
+
JavaScript bindings are experimental implementations, and neither claims
|
|
27
|
+
full conformance yet.
|
|
28
|
+
|
|
29
|
+
## Get started
|
|
30
|
+
|
|
31
|
+
You need Python 3.14 or later; *hexa* has no dependencies.
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
python3 -m pip install -e .
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
The Python distribution is named `hexa-pipelines`; imports use `hexa`.
|
|
38
|
+
The JavaScript binding uses ES2022 modules without runtime dependencies.
|
|
39
|
+
From a checkout, run `bun run example` or
|
|
40
|
+
`node examples/mortgage-people/run.js`; import its public API from
|
|
41
|
+
`hexa/shapes.js`, `hexa/combinators.js` and `hexa/runtime.js`.
|
|
42
|
+
|
|
43
|
+
To test or prepare a release from a checkout, install `uv` and run:
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
make test
|
|
47
|
+
make test-js # JavaScript tests; requires Bun.
|
|
48
|
+
make test-all # Both bindings.
|
|
49
|
+
make test-release # Build the source distribution and wheel.
|
|
50
|
+
make release # Test, build, and publish to PyPI.
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## In a nutshell
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import asyncio, json
|
|
57
|
+
from hexa.shapes import Text, Int, Positive, either, loose, slots, codec
|
|
58
|
+
from hexa.combinators import pipe, select, each, project, collect, Policy
|
|
59
|
+
from hexa.runtime import run
|
|
60
|
+
|
|
61
|
+
# Canonical values, and the one place a source format is converted.
|
|
62
|
+
CountryCode = either("AU", "NZ")
|
|
63
|
+
CountryName = codec("CountryName", source=Text, canonical=CountryCode,
|
|
64
|
+
map={"AU": "Australia", "NZ": ["New Zealand", "Aotearoa New Zealand"]})
|
|
65
|
+
|
|
66
|
+
# Slots: named, typed capture points.
|
|
67
|
+
Client = slots("Client", Id=Int & Positive, Name=Text, Country=CountryCode)
|
|
68
|
+
|
|
69
|
+
# What to match in the source (`loose` allows extra keys)...
|
|
70
|
+
ClientRow = loose({"id": Client.Id, "name": Client.Name,
|
|
71
|
+
"country": Client.Country @ CountryName})
|
|
72
|
+
# ...and the record to fill from the captures.
|
|
73
|
+
PersonRecord = {"Name": Client.Name, "Country": Client.Country}
|
|
74
|
+
|
|
75
|
+
# Select every client, project each row, drop the ones that do not fit.
|
|
76
|
+
PeopleExtract = collect(pipe(select("clients/*"),
|
|
77
|
+
each(project(ClientRow, PersonRecord), Policy.Drop)))
|
|
78
|
+
|
|
79
|
+
data = {"clients": [
|
|
80
|
+
{"id": 7, "name": "Aroha", "country": "Aotearoa New Zealand"},
|
|
81
|
+
{"id": 8, "name": "Mere", "country": "Narnia"},
|
|
82
|
+
{"id": -3, "name": "Sam", "country": "Australia"}]}
|
|
83
|
+
print(json.dumps(asyncio.run(run({}, PeopleExtract, data)), indent=1))
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The result holds one record, `{"Name": "Aroha", "Country": "NZ"}`, and a
|
|
87
|
+
ledger with two gaps: `CountryName.Unmapped` at `/clients/1` and
|
|
88
|
+
`Issue.Constraint` on `id` at `/clients/2`.
|
|
89
|
+
|
|
90
|
+
`Policy.Drop` is a *policy*: the declared response when a stage hits a
|
|
91
|
+
recoverable failure, such as a row that does not match its shape or a value
|
|
92
|
+
that cannot be mapped. `Policy.Abort`, the default for `each`, stops the run,
|
|
93
|
+
`Policy.Drop` skips the item and records it in the ledger,
|
|
94
|
+
`Policy.Substitute(Value)` uses a fixed value instead, and
|
|
95
|
+
`Policy.Fallback(Stage)` runs another stage. For a full program that fetches
|
|
96
|
+
its data through effects, with fixtures and a recorded journal, see
|
|
97
|
+
[examples/mortgage-people](examples/README.md).
|
|
98
|
+
|
|
99
|
+
## Features
|
|
100
|
+
|
|
101
|
+
- *Shapes and slots*: match structures, capture typed values into named
|
|
102
|
+
slots, and get value-free issues for what did not match.
|
|
103
|
+
- *Templates and projection*: expand captures into fresh records; a shape and
|
|
104
|
+
a template together are a function from one structure to another.
|
|
105
|
+
- *Codecs*: convert between a source format and a canonical value (dates,
|
|
106
|
+
decimals, lookup tables), in both directions.
|
|
107
|
+
- *Combinators*: `pipe`, `each`, `choose`, `guard`, `recover`, `retry`,
|
|
108
|
+
`parallel`, `always` and more, building stages from shapes and policies.
|
|
109
|
+
- *Effects as data*: stages yield `{operation, arguments}` and the runtime
|
|
110
|
+
performs them, so a run can be journaled, replayed, or tested with `inert`.
|
|
111
|
+
- *Ledger*: gaps and losses are reported per row with their reason and
|
|
112
|
+
location, and summarised as coverage.
|
|
113
|
+
- *Profiles and requests*: callers send shapes as JSON, checked against what
|
|
114
|
+
the host allows them to reference and how large they may be.
|
|
115
|
+
|
|
116
|
+
## Design
|
|
117
|
+
|
|
118
|
+
A program is a set of declarations read in one direction: output template,
|
|
119
|
+
slots, codecs, source shapes, then combinators. Hand-written stages are
|
|
120
|
+
generators, used only to yield effects or fan in earlier results.
|
|
121
|
+
|
|
122
|
+
- *Shapes never convert.* Conversion happens only in codecs, so every format
|
|
123
|
+
change has a name and a reverse. The cost is declaring a codec where an
|
|
124
|
+
inline expression would have been shorter.
|
|
125
|
+
- *Gaps are reported, never returned.* A missing or unmappable value becomes a
|
|
126
|
+
ledger entry and a policy decides whether the row is dropped, substituted or
|
|
127
|
+
aborts the run. You never get a half-filled record by accident, but you must
|
|
128
|
+
choose a policy for each place data may be absent.
|
|
129
|
+
- *Effects are inert.* Because stages describe I/O rather than perform it,
|
|
130
|
+
replay and testing need no mocks. The cost is that effectful code is written
|
|
131
|
+
as generators and cannot call native awaitables directly.
|
|
132
|
+
- *Configuration is not data.* Program code references codes and enum members
|
|
133
|
+
as symbols (`Policy.Drop`), and only the runtime turns them into strings in
|
|
134
|
+
outputs. A misspelt member fails at import, not on the first bad row.
|
|
135
|
+
- *One vocabulary across languages.* Every binding uses the same canonical
|
|
136
|
+
names and data keys; only syntax differs. This rules out names that clash
|
|
137
|
+
with a builtin in any target language.
|
|
138
|
+
|
|
139
|
+
Known limits: selectors have no index access, callers cannot send templates,
|
|
140
|
+
profiles do not compose, and requests are JSON only. Each is a deliberate
|
|
141
|
+
deferral, explained in
|
|
142
|
+
[spec-004](docs/specs/spec-004-shape_pipelines_shapes.md#trade-offs-and-decisions).
|
|
143
|
+
|
|
144
|
+
## Where to go next
|
|
145
|
+
|
|
146
|
+
- [docs/specs/](docs/specs/): the specification. Start with
|
|
147
|
+
[spec-001](docs/specs/spec-001-shape_pipelines.md) for concepts and naming,
|
|
148
|
+
then spec-004 (shapes) and spec-005 (stages); spec-002 and spec-003 cover
|
|
149
|
+
the JavaScript and Python bindings.
|
|
150
|
+
- [docs/skills/shape-pipelines/](docs/skills/shape-pipelines/SKILL.md):
|
|
151
|
+
working rules for writing programs in this style, also usable as an agent
|
|
152
|
+
skill.
|
|
153
|
+
- [examples/](examples/README.md): the mortgage-people pipeline in both
|
|
154
|
+
bindings; run it with
|
|
155
|
+
`python3 examples/mortgage-people/people_extract.py` or `bun run example`.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "hexa-pipelines"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Shape-directed pipelines with inert effects and replay"
|
|
9
|
+
requires-python = ">=3.14"
|
|
10
|
+
dependencies = []
|
|
11
|
+
|
|
12
|
+
[tool.setuptools.packages.find]
|
|
13
|
+
where = ["src/py"]
|
|
14
|
+
include = ["hexa*"]
|
|
15
|
+
|
|
16
|
+
[tool.ruff]
|
|
17
|
+
line-length = 160
|
|
18
|
+
|
|
19
|
+
[tool.ruff.format]
|
|
20
|
+
indent-style = "tab"
|
|
21
|
+
|
|
22
|
+
[tool.ruff.lint]
|
|
23
|
+
select = ["E", "F", "W", "N"]
|
|
24
|
+
# NOTE: House style indents with tabs, keeps long declarative lines and names
|
|
25
|
+
# fields camelCase. The spec (spec-001, spec-003) names stages, operations and
|
|
26
|
+
# their parameters in PascalCase, `stages` in lowercase and `Failure` without an
|
|
27
|
+
# `Error` suffix, and stage namespaces define methods without `self`.
|
|
28
|
+
ignore = ["W191", "E501", "N801", "N802", "N803", "N805", "N806", "N815", "N816", "N818"]
|
|
29
|
+
|
|
30
|
+
[tool.pytest.ini_options]
|
|
31
|
+
testpaths = ["tests"]
|
|
32
|
+
pythonpath = ["src/py"]
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Module: hexa.codec — Declare table and code codecs, optionally parameterised."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .core import Gap, codeNamespace, freeze, register
|
|
6
|
+
from .shape import Codec, CodecFamily, compileShape, errors
|
|
7
|
+
|
|
8
|
+
# -----------------------------------------------------------------------------
|
|
9
|
+
#
|
|
10
|
+
# DEFINITIONS
|
|
11
|
+
#
|
|
12
|
+
# -----------------------------------------------------------------------------
|
|
13
|
+
|
|
14
|
+
__all__ = ["codec"]
|
|
15
|
+
|
|
16
|
+
# -----------------------------------------------------------------------------
|
|
17
|
+
#
|
|
18
|
+
# UTILITIES
|
|
19
|
+
#
|
|
20
|
+
# -----------------------------------------------------------------------------
|
|
21
|
+
|
|
22
|
+
def tableFunctions(table, gaps, source, canonical, reasons):
|
|
23
|
+
"""Build the parse and format functions of a table codec."""
|
|
24
|
+
reverse, forward = {}, {}
|
|
25
|
+
for key, sources in table.items():
|
|
26
|
+
if not isinstance(key, str) or errors(key, canonical):
|
|
27
|
+
raise ValueError("invalid canonical table key")
|
|
28
|
+
items = sources if isinstance(sources, list) else [sources]
|
|
29
|
+
if not items:
|
|
30
|
+
raise ValueError("empty source alternatives")
|
|
31
|
+
for item in items:
|
|
32
|
+
if not isinstance(item, str) or errors(item, source) or item in reverse:
|
|
33
|
+
raise ValueError("invalid or duplicate table source")
|
|
34
|
+
reverse[item] = key
|
|
35
|
+
forward[key] = items[0]
|
|
36
|
+
for item in gaps:
|
|
37
|
+
if item in reverse or not isinstance(item, str):
|
|
38
|
+
raise ValueError("duplicate or invalid gap source")
|
|
39
|
+
|
|
40
|
+
def parse(value):
|
|
41
|
+
if value in gaps:
|
|
42
|
+
raise Gap(getattr(reasons, gaps[value]))
|
|
43
|
+
if value not in reverse:
|
|
44
|
+
raise Gap(reasons.Unmapped)
|
|
45
|
+
return reverse[value]
|
|
46
|
+
|
|
47
|
+
def format(value, original=None):
|
|
48
|
+
if original in reverse and reverse[original] == value:
|
|
49
|
+
return original
|
|
50
|
+
if value not in forward:
|
|
51
|
+
raise Gap(reasons.Unformattable)
|
|
52
|
+
return forward[value]
|
|
53
|
+
|
|
54
|
+
return parse, format
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# -----------------------------------------------------------------------------
|
|
58
|
+
#
|
|
59
|
+
# HIGH-LEVEL API
|
|
60
|
+
#
|
|
61
|
+
# -----------------------------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
def codec(name, *, source, canonical, parse=None, format=None, map=None, gaps=(), losses=(), params=None, examples=()):
|
|
64
|
+
"""Declare a table or pure code codec, optionally parameterised."""
|
|
65
|
+
source, canonical = compileShape(source), compileShape(canonical)
|
|
66
|
+
if map is not None and gaps == ():
|
|
67
|
+
gaps = {}
|
|
68
|
+
if map is not None and not isinstance(gaps, dict):
|
|
69
|
+
raise TypeError("table gaps must map source text to reason names")
|
|
70
|
+
gapNames = tuple(dict.fromkeys(("Unmapped", "Unformattable", *(gaps.values() if isinstance(gaps, dict) else gaps))))
|
|
71
|
+
reasons = codeNamespace(name, (*gapNames, *losses))
|
|
72
|
+
if map is not None:
|
|
73
|
+
if parse is not None or format is not None or params or losses:
|
|
74
|
+
raise TypeError("table codec cannot also define code, parameters or losses")
|
|
75
|
+
parse, format = tableFunctions(map, dict(gaps), source, canonical, reasons)
|
|
76
|
+
if not callable(parse) or not callable(format):
|
|
77
|
+
raise TypeError("codecs require parse and format functions")
|
|
78
|
+
gapMembers = tuple(reasons.members[item] for item in gapNames)
|
|
79
|
+
lossMembers = tuple(reasons.members[item] for item in losses)
|
|
80
|
+
frozenExamples = tuple(freeze(item) for item in examples)
|
|
81
|
+
if params:
|
|
82
|
+
parameters = tuple((key, compileShape(value)) for key, value in params.items())
|
|
83
|
+
return register(name, CodecFamily(name, source, canonical, parse, format, gapMembers, lossMembers, parameters, frozenExamples))
|
|
84
|
+
return register(name, Codec(name, source, canonical, parse, format, gapMembers, lossMembers,
|
|
85
|
+
table=freeze(map) if map is not None else None, gapTable=freeze(gaps) if isinstance(gaps, dict) else None,
|
|
86
|
+
examples=frozenExamples))
|
|
87
|
+
|
|
88
|
+
# EOF
|