hexa-pipelines 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. hexa_pipelines-0.2.0/LICENSE.md +27 -0
  2. hexa_pipelines-0.2.0/PKG-INFO +7 -0
  3. hexa_pipelines-0.2.0/README.md +155 -0
  4. hexa_pipelines-0.2.0/pyproject.toml +32 -0
  5. hexa_pipelines-0.2.0/setup.cfg +4 -0
  6. hexa_pipelines-0.2.0/src/py/hexa/__init__.py +7 -0
  7. hexa_pipelines-0.2.0/src/py/hexa/codec.py +88 -0
  8. hexa_pipelines-0.2.0/src/py/hexa/combinators.py +466 -0
  9. hexa_pipelines-0.2.0/src/py/hexa/core.py +405 -0
  10. hexa_pipelines-0.2.0/src/py/hexa/encoding.py +307 -0
  11. hexa_pipelines-0.2.0/src/py/hexa/instructions.py +123 -0
  12. hexa_pipelines-0.2.0/src/py/hexa/profile.py +280 -0
  13. hexa_pipelines-0.2.0/src/py/hexa/runtime.py +648 -0
  14. hexa_pipelines-0.2.0/src/py/hexa/selection_request.py +187 -0
  15. hexa_pipelines-0.2.0/src/py/hexa/selector.py +77 -0
  16. hexa_pipelines-0.2.0/src/py/hexa/shape.py +872 -0
  17. hexa_pipelines-0.2.0/src/py/hexa/shapes.py +40 -0
  18. hexa_pipelines-0.2.0/src/py/hexa/stage.py +114 -0
  19. hexa_pipelines-0.2.0/src/py/hexa/template.py +235 -0
  20. hexa_pipelines-0.2.0/src/py/hexa/vocabulary.py +142 -0
  21. hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/PKG-INFO +7 -0
  22. hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/SOURCES.txt +29 -0
  23. hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/dependency_links.txt +1 -0
  24. hexa_pipelines-0.2.0/src/py/hexa_pipelines.egg-info/top_level.txt +1 -0
  25. hexa_pipelines-0.2.0/tests/test_combinators.py +241 -0
  26. hexa_pipelines-0.2.0/tests/test_config_review_regressions.py +73 -0
  27. hexa_pipelines-0.2.0/tests/test_conformance.py +84 -0
  28. hexa_pipelines-0.2.0/tests/test_profiles.py +215 -0
  29. hexa_pipelines-0.2.0/tests/test_runtime.py +601 -0
  30. hexa_pipelines-0.2.0/tests/test_shapes.py +299 -0
  31. hexa_pipelines-0.2.0/tests/test_structure_parity.py +133 -0
@@ -0,0 +1,27 @@
1
+ # Revised BSD License
2
+
3
+ Copyright (c) 2026, Sébastien Pierre. All rights reserved.
4
+
5
+ Redistribution and use in source and binary forms, with or without modification,
6
+ are permitted provided that the following conditions are met:
7
+
8
+ Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer. Redistributions in binary
10
+ form must reproduce the above copyright notice, this list of conditions and
11
+ the following disclaimer in the documentation and/or other materials
12
+ provided with the distribution. Neither the name of Sébastien Pierre nor
13
+ the names of its contributors may be used to endorse or
14
+ promote products derived from this software without specific prior written
15
+ permission.
16
+
17
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,7 @@
1
+ Metadata-Version: 2.4
2
+ Name: hexa-pipelines
3
+ Version: 0.2.0
4
+ Summary: Shape-directed pipelines with inert effects and replay
5
+ Requires-Python: >=3.14
6
+ License-File: LICENSE.md
7
+ Dynamic: license-file
@@ -0,0 +1,155 @@
1
+ ```
2
+ ▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄
3
+ █ ░▓░ ▓ █ ▄▀░▓░ ▓ █ ░▓░ ▓ █ ▄▀░▓░ ▀▄
4
+ █ ▒ ▒ █ ▐▌ ▒ ▒ ▐▌ ▒ ▒ ▐▌ ▐▌ ▒ ▐▌
5
+ █ ░ ░ █ █ ░ ▄▄▄▄▄▄░ █ ░ ░ █ ▒ ░ █ ▓
6
+ █ ▄▓ █ █ █ ▀▄ ▓ ▄▀ ▒ ▒
7
+ ▓ █ ▓ █ ▄▀ ▀▄ ░░ ░ ░
8
+ ▒ █▀ ▓ ▓ ▀▀▀▀▀▀▀ ▐▌ █▀ ▐▌ ▒▒ ▒ ░░
9
+ ▓░ ▓ ▒ ▒░ ▓ ▓░ ▓ ▓ ▒▓░ ▓ ░▒░
10
+ ▓▓▒░ ▓ █ ▐▓▒░ ▒ ▓▓▒░ ▓ ▒ ▓▓▒░ ░█ ░▒▓▒
11
+ ██▓▒░ ▒ █ ▀▓▒░ ░ ██▓▒░ ▒ ░ ██▓▒░ ▒█ ░▒▓█▓
12
+ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀
13
+ ```
14
+
15
+ *hexa* is a library for writing data pipelines as projections between
16
+ shapes. You declare what to match in the source data (a *shape*, the regex of
17
+ data structures), what to build from it (a *template*), and compose the two
18
+ into stages that select, project, recover and retry, with I/O yielded as
19
+ inert effects that the runtime performs, records and replays.
20
+
21
+ It comes from extraction work: pulling records out of CRM and partner APIs,
22
+ normalising their formats, and reporting what did not fit instead of failing
23
+ or silently guessing. In our view most of such a program is configuration,
24
+ so *hexa* makes the shapes, codecs and pipeline declaration the program, and
25
+ hand-written code the exception. It is a specification first: the Python and
26
+ JavaScript bindings are experimental implementations, and neither claims
27
+ full conformance yet.
28
+
29
+ ## Get started
30
+
31
+ You need Python 3.14 or later; *hexa* has no dependencies.
32
+
33
+ ```sh
34
+ python3 -m pip install -e .
35
+ ```
36
+
37
+ The Python distribution is named `hexa-pipelines`; imports use `hexa`.
38
+ The JavaScript binding uses ES2022 modules without runtime dependencies.
39
+ From a checkout, run `bun run example` or
40
+ `node examples/mortgage-people/run.js`; import its public API from
41
+ `hexa/shapes.js`, `hexa/combinators.js` and `hexa/runtime.js`.
42
+
43
+ To test or prepare a release from a checkout, install `uv` and run:
44
+
45
+ ```sh
46
+ make test
47
+ make test-js # JavaScript tests; requires Bun.
48
+ make test-all # Both bindings.
49
+ make test-release # Build the source distribution and wheel.
50
+ make release # Test, build, and publish to PyPI.
51
+ ```
52
+
53
+ ## In a nutshell
54
+
55
+ ```python
56
+ import asyncio, json
57
+ from hexa.shapes import Text, Int, Positive, either, loose, slots, codec
58
+ from hexa.combinators import pipe, select, each, project, collect, Policy
59
+ from hexa.runtime import run
60
+
61
+ # Canonical values, and the one place a source format is converted.
62
+ CountryCode = either("AU", "NZ")
63
+ CountryName = codec("CountryName", source=Text, canonical=CountryCode,
64
+ map={"AU": "Australia", "NZ": ["New Zealand", "Aotearoa New Zealand"]})
65
+
66
+ # Slots: named, typed capture points.
67
+ Client = slots("Client", Id=Int & Positive, Name=Text, Country=CountryCode)
68
+
69
+ # What to match in the source (`loose` allows extra keys)...
70
+ ClientRow = loose({"id": Client.Id, "name": Client.Name,
71
+ "country": Client.Country @ CountryName})
72
+ # ...and the record to fill from the captures.
73
+ PersonRecord = {"Name": Client.Name, "Country": Client.Country}
74
+
75
+ # Select every client, project each row, drop the ones that do not fit.
76
+ PeopleExtract = collect(pipe(select("clients/*"),
77
+ each(project(ClientRow, PersonRecord), Policy.Drop)))
78
+
79
+ data = {"clients": [
80
+ {"id": 7, "name": "Aroha", "country": "Aotearoa New Zealand"},
81
+ {"id": 8, "name": "Mere", "country": "Narnia"},
82
+ {"id": -3, "name": "Sam", "country": "Australia"}]}
83
+ print(json.dumps(asyncio.run(run({}, PeopleExtract, data)), indent=1))
84
+ ```
85
+
86
+ The result holds one record, `{"Name": "Aroha", "Country": "NZ"}`, and a
87
+ ledger with two gaps: `CountryName.Unmapped` at `/clients/1` and
88
+ `Issue.Constraint` on `id` at `/clients/2`.
89
+
90
+ `Policy.Drop` is a *policy*: the declared response when a stage hits a
91
+ recoverable failure, such as a row that does not match its shape or a value
92
+ that cannot be mapped. `Policy.Abort`, the default for `each`, stops the run,
93
+ `Policy.Drop` skips the item and records it in the ledger,
94
+ `Policy.Substitute(Value)` uses a fixed value instead, and
95
+ `Policy.Fallback(Stage)` runs another stage. For a full program that fetches
96
+ its data through effects, with fixtures and a recorded journal, see
97
+ [examples/mortgage-people](examples/README.md).
98
+
99
+ ## Features
100
+
101
+ - *Shapes and slots*: match structures, capture typed values into named
102
+ slots, and get value-free issues for what did not match.
103
+ - *Templates and projection*: expand captures into fresh records; a shape and
104
+ a template together are a function from one structure to another.
105
+ - *Codecs*: convert between a source format and a canonical value (dates,
106
+ decimals, lookup tables), in both directions.
107
+ - *Combinators*: `pipe`, `each`, `choose`, `guard`, `recover`, `retry`,
108
+ `parallel`, `always` and more, building stages from shapes and policies.
109
+ - *Effects as data*: stages yield `{operation, arguments}` and the runtime
110
+ performs them, so a run can be journaled, replayed, or tested with `inert`.
111
+ - *Ledger*: gaps and losses are reported per row with their reason and
112
+ location, and summarised as coverage.
113
+ - *Profiles and requests*: callers send shapes as JSON, checked against what
114
+ the host allows them to reference and how large they may be.
115
+
116
+ ## Design
117
+
118
+ A program is a set of declarations read in one direction: output template,
119
+ slots, codecs, source shapes, then combinators. Hand-written stages are
120
+ generators, used only to yield effects or fan in earlier results.
121
+
122
+ - *Shapes never convert.* Conversion happens only in codecs, so every format
123
+ change has a name and a reverse. The cost is declaring a codec where an
124
+ inline expression would have been shorter.
125
+ - *Gaps are reported, never returned.* A missing or unmappable value becomes a
126
+ ledger entry and a policy decides whether the row is dropped, substituted or
127
+ aborts the run. You never get a half-filled record by accident, but you must
128
+ choose a policy for each place data may be absent.
129
+ - *Effects are inert.* Because stages describe I/O rather than perform it,
130
+ replay and testing need no mocks. The cost is that effectful code is written
131
+ as generators and cannot call native awaitables directly.
132
+ - *Configuration is not data.* Program code references codes and enum members
133
+ as symbols (`Policy.Drop`), and only the runtime turns them into strings in
134
+ outputs. A misspelt member fails at import, not on the first bad row.
135
+ - *One vocabulary across languages.* Every binding uses the same canonical
136
+ names and data keys; only syntax differs. This rules out names that clash
137
+ with a builtin in any target language.
138
+
139
+ Known limits: selectors have no index access, callers cannot send templates,
140
+ profiles do not compose, and requests are JSON only. Each is a deliberate
141
+ deferral, explained in
142
+ [spec-004](docs/specs/spec-004-shape_pipelines_shapes.md#trade-offs-and-decisions).
143
+
144
+ ## Where to go next
145
+
146
+ - [docs/specs/](docs/specs/): the specification. Start with
147
+ [spec-001](docs/specs/spec-001-shape_pipelines.md) for concepts and naming,
148
+ then spec-004 (shapes) and spec-005 (stages); spec-002 and spec-003 cover
149
+ the JavaScript and Python bindings.
150
+ - [docs/skills/shape-pipelines/](docs/skills/shape-pipelines/SKILL.md):
151
+ working rules for writing programs in this style, also usable as an agent
152
+ skill.
153
+ - [examples/](examples/README.md): the mortgage-people pipeline in both
154
+ bindings; run it with
155
+ `python3 examples/mortgage-people/people_extract.py` or `bun run example`.
@@ -0,0 +1,32 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "hexa-pipelines"
7
+ version = "0.2.0"
8
+ description = "Shape-directed pipelines with inert effects and replay"
9
+ requires-python = ">=3.14"
10
+ dependencies = []
11
+
12
+ [tool.setuptools.packages.find]
13
+ where = ["src/py"]
14
+ include = ["hexa*"]
15
+
16
+ [tool.ruff]
17
+ line-length = 160
18
+
19
+ [tool.ruff.format]
20
+ indent-style = "tab"
21
+
22
+ [tool.ruff.lint]
23
+ select = ["E", "F", "W", "N"]
24
+ # NOTE: House style indents with tabs, keeps long declarative lines and names
25
+ # fields camelCase. The spec (spec-001, spec-003) names stages, operations and
26
+ # their parameters in PascalCase, `stages` in lowercase and `Failure` without an
27
+ # `Error` suffix, and stage namespaces define methods without `self`.
28
+ ignore = ["W191", "E501", "N801", "N802", "N803", "N805", "N806", "N815", "N816", "N818"]
29
+
30
+ [tool.pytest.ini_options]
31
+ testpaths = ["tests"]
32
+ pythonpath = ["src/py"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,7 @@
1
+ """Module: hexa — Shape-directed pipelines for Python."""
2
+
3
+ from __future__ import annotations
4
+
5
+ # NOTE: Import from hexa.shapes, hexa.combinators and hexa.runtime explicitly.
6
+
7
+ # EOF
@@ -0,0 +1,88 @@
1
+ """Module: hexa.codec — Declare table and code codecs, optionally parameterised."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .core import Gap, codeNamespace, freeze, register
6
+ from .shape import Codec, CodecFamily, compileShape, errors
7
+
8
+ # -----------------------------------------------------------------------------
9
+ #
10
+ # DEFINITIONS
11
+ #
12
+ # -----------------------------------------------------------------------------
13
+
14
+ __all__ = ["codec"]
15
+
16
+ # -----------------------------------------------------------------------------
17
+ #
18
+ # UTILITIES
19
+ #
20
+ # -----------------------------------------------------------------------------
21
+
22
+ def tableFunctions(table, gaps, source, canonical, reasons):
23
+ """Build the parse and format functions of a table codec."""
24
+ reverse, forward = {}, {}
25
+ for key, sources in table.items():
26
+ if not isinstance(key, str) or errors(key, canonical):
27
+ raise ValueError("invalid canonical table key")
28
+ items = sources if isinstance(sources, list) else [sources]
29
+ if not items:
30
+ raise ValueError("empty source alternatives")
31
+ for item in items:
32
+ if not isinstance(item, str) or errors(item, source) or item in reverse:
33
+ raise ValueError("invalid or duplicate table source")
34
+ reverse[item] = key
35
+ forward[key] = items[0]
36
+ for item in gaps:
37
+ if item in reverse or not isinstance(item, str):
38
+ raise ValueError("duplicate or invalid gap source")
39
+
40
+ def parse(value):
41
+ if value in gaps:
42
+ raise Gap(getattr(reasons, gaps[value]))
43
+ if value not in reverse:
44
+ raise Gap(reasons.Unmapped)
45
+ return reverse[value]
46
+
47
+ def format(value, original=None):
48
+ if original in reverse and reverse[original] == value:
49
+ return original
50
+ if value not in forward:
51
+ raise Gap(reasons.Unformattable)
52
+ return forward[value]
53
+
54
+ return parse, format
55
+
56
+
57
+ # -----------------------------------------------------------------------------
58
+ #
59
+ # HIGH-LEVEL API
60
+ #
61
+ # -----------------------------------------------------------------------------
62
+
63
+ def codec(name, *, source, canonical, parse=None, format=None, map=None, gaps=(), losses=(), params=None, examples=()):
64
+ """Declare a table or pure code codec, optionally parameterised."""
65
+ source, canonical = compileShape(source), compileShape(canonical)
66
+ if map is not None and gaps == ():
67
+ gaps = {}
68
+ if map is not None and not isinstance(gaps, dict):
69
+ raise TypeError("table gaps must map source text to reason names")
70
+ gapNames = tuple(dict.fromkeys(("Unmapped", "Unformattable", *(gaps.values() if isinstance(gaps, dict) else gaps))))
71
+ reasons = codeNamespace(name, (*gapNames, *losses))
72
+ if map is not None:
73
+ if parse is not None or format is not None or params or losses:
74
+ raise TypeError("table codec cannot also define code, parameters or losses")
75
+ parse, format = tableFunctions(map, dict(gaps), source, canonical, reasons)
76
+ if not callable(parse) or not callable(format):
77
+ raise TypeError("codecs require parse and format functions")
78
+ gapMembers = tuple(reasons.members[item] for item in gapNames)
79
+ lossMembers = tuple(reasons.members[item] for item in losses)
80
+ frozenExamples = tuple(freeze(item) for item in examples)
81
+ if params:
82
+ parameters = tuple((key, compileShape(value)) for key, value in params.items())
83
+ return register(name, CodecFamily(name, source, canonical, parse, format, gapMembers, lossMembers, parameters, frozenExamples))
84
+ return register(name, Codec(name, source, canonical, parse, format, gapMembers, lossMembers,
85
+ table=freeze(map) if map is not None else None, gapTable=freeze(gaps) if isinstance(gaps, dict) else None,
86
+ examples=frozenExamples))
87
+
88
+ # EOF