interlacing 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interlacing/__init__.py +50 -0
- interlacing/base/__init__.py +25 -0
- interlacing/base/canonical.py +14 -0
- interlacing/base/edge.py +61 -0
- interlacing/base/envelope.py +25 -0
- interlacing/base/graph.py +271 -0
- interlacing/base/method.py +21 -0
- interlacing/base/node.py +151 -0
- interlacing/base/port.py +87 -0
- interlacing/base/protocol.py +90 -0
- interlacing/base/schema.py +176 -0
- interlacing/blocks/__init__.py +107 -0
- interlacing/cli.py +154 -0
- interlacing/engine/__init__.py +8 -0
- interlacing/engine/engine.py +1062 -0
- interlacing/engine/journal.py +97 -0
- interlacing/engine/stream.py +42 -0
- interlacing/engine/transport.py +44 -0
- interlacing/methods/__init__.py +8 -0
- interlacing/methods/chain.py +49 -0
- interlacing/methods/fan.py +68 -0
- interlacing/methods/loop.py +63 -0
- interlacing/methods/pool.py +78 -0
- interlacing/methods/tree.py +117 -0
- interlacing/models/__init__.py +9 -0
- interlacing/models/providers.py +511 -0
- interlacing/nodes/__init__.py +10 -0
- interlacing/nodes/agent.py +242 -0
- interlacing/nodes/flow.py +185 -0
- interlacing/nodes/function.py +44 -0
- interlacing/nodes/graph.py +49 -0
- interlacing/nodes/outside.py +110 -0
- interlacing/nodes/process.py +127 -0
- interlacing/protocols/__init__.py +24 -0
- interlacing/protocols/artifact.py +44 -0
- interlacing/protocols/file.py +197 -0
- interlacing/protocols/standard.py +54 -0
- interlacing/protocols/store.py +92 -0
- interlacing/protocols/value.py +7 -0
- interlacing/py.typed +0 -0
- interlacing/runs.py +307 -0
- interlacing/serve.py +190 -0
- interlacing/spec/__init__.py +6 -0
- interlacing/spec/catalog.py +111 -0
- interlacing/spec/kinds.py +167 -0
- interlacing/spec/load.py +194 -0
- interlacing/spec/spawn.py +129 -0
- interlacing-0.11.0.dist-info/METADATA +203 -0
- interlacing-0.11.0.dist-info/RECORD +52 -0
- interlacing-0.11.0.dist-info/WHEEL +4 -0
- interlacing-0.11.0.dist-info/entry_points.txt +2 -0
- interlacing-0.11.0.dist-info/licenses/LICENSE +21 -0
interlacing/__init__.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""interlacing: nodes, edges and methods, and an engine that runs the graphs they make.
|
|
2
|
+
|
|
3
|
+
A node is anything that takes information in and gives information out: a model, a function, an external
|
|
4
|
+
program, a device, a person, another graph. An edge carries information from one node to another under a
|
|
5
|
+
written protocol. A method is a reusable way of joining nodes: a chain, a loop, a fan, a tree.
|
|
6
|
+
|
|
7
|
+
Layers, bottom up (each imports only from below):
|
|
8
|
+
|
|
9
|
+
1. base Node, Port, Schema, Edge, Protocol, Envelope, Graph, GraphBuilder, Method
|
|
10
|
+
2. engine Engine, Result, Journal, Transport
|
|
11
|
+
3. parts nodes, protocols, methods, models: the ones that come with the package
|
|
12
|
+
blocks a designed graph kept as a unit and used as often as needed: Block, seq, par
|
|
13
|
+
4. spec graphs written as data: dump, load, and the Catalog that says what data may reach
|
|
14
|
+
5. runs, cli, serve a run kept in a directory, the ``interlacing`` command that works on one, and the
|
|
15
|
+
same operations offered to an MCP client
|
|
16
|
+
|
|
17
|
+
Public names are importable from the package root. Standard library only.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
__version__ = "0.11.0"
|
|
21
|
+
|
|
22
|
+
_PUBLIC = {
|
|
23
|
+
# base
|
|
24
|
+
"Node": "base", "Port": "base", "Firing": "base", "Context": "base", "Schema": "base", "Edge": "base",
|
|
25
|
+
"Invalid": "base", "Protocol": "base", "Protocols": "base", "Delivery": "base", "Envelope": "base", "Ref": "base",
|
|
26
|
+
"Graph": "base", "GraphBuilder": "base", "GraphError": "base", "NodeError": "base", "Method": "base",
|
|
27
|
+
# engine
|
|
28
|
+
"Engine": "engine", "Result": "engine", "RunError": "engine", "OverBudget": "engine", "Journal": "engine", "Transport": "engine",
|
|
29
|
+
# parts
|
|
30
|
+
"FunctionNode": "nodes", "AgentNode": "nodes", "Tool": "nodes", "ProcessNode": "nodes", "SourceNode": "nodes",
|
|
31
|
+
"SinkNode": "nodes", "HumanNode": "nodes", "GraphNode": "nodes", "Desk": "nodes",
|
|
32
|
+
"Chain": "methods", "Loop": "methods", "Fan": "methods", "Tree": "methods", "Star": "methods",
|
|
33
|
+
"Governed": "methods", "Pool": "methods",
|
|
34
|
+
"Block": "blocks", "seq": "blocks", "par": "blocks",
|
|
35
|
+
"DEFAULT": "protocols", "standard": "protocols", "File": "protocols", "FileOf": "protocols",
|
|
36
|
+
"with_files": "protocols",
|
|
37
|
+
"Provider": "models", "FakeProvider": "models", "ChatResult": "models",
|
|
38
|
+
# graphs as data
|
|
39
|
+
"Catalog": "spec", "SpecError": "spec", "SpawnNode": "spec", "dump": "spec", "load": "spec",
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def __getattr__(name):
|
|
44
|
+
if name in _PUBLIC:
|
|
45
|
+
import importlib
|
|
46
|
+
return getattr(importlib.import_module(f"{__name__}.{_PUBLIC[name]}"), name)
|
|
47
|
+
raise AttributeError(name)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
__all__ = ["__version__", *_PUBLIC]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""The abstractions everything else is built on. No behaviour of any particular node, edge or method lives here.
|
|
2
|
+
|
|
3
|
+
Node, Port, Schema what takes information in and gives it out, and what its ports carry
|
|
4
|
+
Edge, Protocol a flow between two ports, and the rules of that flow
|
|
5
|
+
Envelope, Ref the form in which information travels
|
|
6
|
+
Graph, GraphBuilder nodes joined by edges, checked once, then read-only
|
|
7
|
+
Method a reusable way of joining nodes
|
|
8
|
+
|
|
9
|
+
This package imports nothing else from ``interlacing``.
|
|
10
|
+
"""
|
|
11
|
+
from .canonical import canonical, digest
|
|
12
|
+
from .edge import Edge, EdgeError, Endpoint
|
|
13
|
+
from .envelope import Envelope, Ref
|
|
14
|
+
from .graph import Graph, GraphBuilder, GraphError
|
|
15
|
+
from .method import Method
|
|
16
|
+
from .node import Context, Firing, Invalid, Node, NodeError
|
|
17
|
+
from .port import Port, PortError
|
|
18
|
+
from .protocol import Delivery, Protocol, ProtocolError, Protocols
|
|
19
|
+
from .schema import ANY, Is, ListOf, Record, Schema, SchemaError, schema
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"ANY", "Context", "Delivery", "Edge", "EdgeError", "Endpoint", "Envelope", "Firing", "Graph", "GraphBuilder",
|
|
23
|
+
"GraphError", "Invalid", "Is", "ListOf", "Method", "Node", "NodeError", "Port", "PortError", "Protocol", "ProtocolError",
|
|
24
|
+
"Protocols", "Record", "Ref", "Schema", "SchemaError", "canonical", "digest", "schema",
|
|
25
|
+
]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""One canonical text form of a definition, and its digest. Two equal definitions always hash the same."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def canonical(value) -> str:
|
|
9
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def digest(value) -> str:
|
|
13
|
+
"""SHA-256 of the canonical form. ``value`` must be plain data: dict, list, str, int, float, bool, None."""
|
|
14
|
+
return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()
|
interlacing/base/edge.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""The edge: one flow of information from an output port to an input port, under a protocol."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import NamedTuple
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class EdgeError(ValueError):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Endpoint(NamedTuple):
|
|
13
|
+
node: str
|
|
14
|
+
port: str
|
|
15
|
+
|
|
16
|
+
def __str__(self):
|
|
17
|
+
return f"{self.node}.{self.port}"
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def of(cls, value) -> "Endpoint":
|
|
21
|
+
"""From an Endpoint, a ``(node, port)`` pair or the text ``"node.port"``."""
|
|
22
|
+
if isinstance(value, str):
|
|
23
|
+
node, dot, port = value.partition(".")
|
|
24
|
+
if not (node and dot and port):
|
|
25
|
+
raise EdgeError(f"expected 'node.port': {value!r}")
|
|
26
|
+
return cls(node, port)
|
|
27
|
+
try:
|
|
28
|
+
node, port = value
|
|
29
|
+
except (TypeError, ValueError):
|
|
30
|
+
raise EdgeError(f"expected (node, port): {value!r}") from None
|
|
31
|
+
return cls(str(node), str(port))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Edge:
|
|
36
|
+
"""``back`` marks an edge that closes a loop: it never counts toward a node's first start, and it must say
|
|
37
|
+
how many rounds it may open at most (``max_rounds``), so no loop can run for ever by construction."""
|
|
38
|
+
|
|
39
|
+
id: str
|
|
40
|
+
source: Endpoint
|
|
41
|
+
target: Endpoint
|
|
42
|
+
protocol: str
|
|
43
|
+
back: bool = False
|
|
44
|
+
max_rounds: int | None = None
|
|
45
|
+
|
|
46
|
+
def __post_init__(self):
|
|
47
|
+
object.__setattr__(self, "source", Endpoint.of(self.source))
|
|
48
|
+
object.__setattr__(self, "target", Endpoint.of(self.target))
|
|
49
|
+
if not isinstance(self.id, str) or not self.id.strip():
|
|
50
|
+
raise EdgeError("an edge needs an id")
|
|
51
|
+
if not isinstance(self.protocol, str) or not self.protocol:
|
|
52
|
+
raise EdgeError(f"edge {self.id}: no protocol")
|
|
53
|
+
if self.back:
|
|
54
|
+
if type(self.max_rounds) is not int or self.max_rounds < 1:
|
|
55
|
+
raise EdgeError(f"edge {self.id}: a back edge needs max_rounds of at least 1")
|
|
56
|
+
elif self.max_rounds is not None:
|
|
57
|
+
raise EdgeError(f"edge {self.id}: only a back edge has max_rounds")
|
|
58
|
+
|
|
59
|
+
def describe(self) -> dict:
|
|
60
|
+
return {"id": self.id, "from": str(self.source), "to": str(self.target), "protocol": self.protocol,
|
|
61
|
+
"back": self.back, "max_rounds": self.max_rounds}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""The envelope: the one form in which information travels along an edge."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class Ref:
|
|
10
|
+
"""A body kept elsewhere (a file, a large value): the envelope and the log carry only this reference."""
|
|
11
|
+
|
|
12
|
+
digest: str # content hash, which is also how the content store finds it
|
|
13
|
+
size: int = 0
|
|
14
|
+
media: str = "" # what it is, when known: "text/markdown", "application/json", ...
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class Envelope:
|
|
19
|
+
id: str
|
|
20
|
+
edge: str # the edge it travels on
|
|
21
|
+
seq: int # its number on that edge, from 1
|
|
22
|
+
round: int # the round of the activation that sent it
|
|
23
|
+
body: Any # small data inline, anything large as a Ref
|
|
24
|
+
digest: str # hash of the body
|
|
25
|
+
cause: tuple[str, ...] = () # the envelopes the sending activation had received: where this came from
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""The graph: nodes joined by edges, checked once and then read-only.
|
|
2
|
+
|
|
3
|
+
``GraphBuilder`` collects nodes and edges; ``compile`` checks the whole and returns a ``Graph`` with its
|
|
4
|
+
indexes built. A graph never changes: a different graph is a new value with a new digest. Building costs
|
|
5
|
+
time proportional to nodes plus edges (plus one walk per back edge); every lookup afterwards is a dictionary
|
|
6
|
+
or tuple access.
|
|
7
|
+
|
|
8
|
+
Rules checked by ``compile``:
|
|
9
|
+
|
|
10
|
+
* every edge joins an existing output port to an existing input port, under a registered protocol that
|
|
11
|
+
accepts the pair;
|
|
12
|
+
* an input that is not ``many`` has at most one forward feed (an edge or a graph input); back edges may
|
|
13
|
+
feed it as well, since they only arrive in later rounds, and so may any number of calls;
|
|
14
|
+
* a required input has a forward feed, otherwise its node could never start;
|
|
15
|
+
* without its back edges the graph has no cycle, and every back edge really closes a loop. With the
|
|
16
|
+
``max_rounds`` every back edge carries, no run can go on for ever.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from collections import deque
|
|
21
|
+
from types import MappingProxyType
|
|
22
|
+
from typing import Iterable, Mapping
|
|
23
|
+
|
|
24
|
+
from .canonical import digest
|
|
25
|
+
from .edge import Edge, EdgeError, Endpoint
|
|
26
|
+
from .node import Firing, Node
|
|
27
|
+
from .protocol import Delivery, ProtocolError, Protocols
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class GraphError(ValueError):
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Graph:
|
|
35
|
+
"""A compiled graph. Construct it with ``GraphBuilder.compile``."""
|
|
36
|
+
|
|
37
|
+
__slots__ = ("nodes", "edges", "inputs", "outputs", "order", "digest", "_node", "_edge", "_out", "_in", "_needs")
|
|
38
|
+
|
|
39
|
+
def __init__(self, nodes, edges, inputs, outputs, order, out_index, in_index, needs):
|
|
40
|
+
self.nodes: tuple[Node, ...] = nodes
|
|
41
|
+
self.edges: tuple[Edge, ...] = edges
|
|
42
|
+
self.inputs: Mapping[str, Endpoint] = MappingProxyType(inputs) # the graph's own inputs, by name
|
|
43
|
+
self.outputs: Mapping[str, Endpoint] = MappingProxyType(outputs) # the graph's own outputs, by name
|
|
44
|
+
self.order: tuple[str, ...] = order # node ids, upstream first (back edges ignored)
|
|
45
|
+
self._node = {node.id: node for node in nodes}
|
|
46
|
+
self._edge = {edge.id: edge for edge in edges}
|
|
47
|
+
self._out, self._in, self._needs = out_index, in_index, needs
|
|
48
|
+
self.digest: str = digest(self.describe())
|
|
49
|
+
|
|
50
|
+
# ---- lookups ---------------------------------------------------------------------------------------
|
|
51
|
+
|
|
52
|
+
def node(self, id: str) -> Node:
|
|
53
|
+
try:
|
|
54
|
+
return self._node[id]
|
|
55
|
+
except KeyError:
|
|
56
|
+
raise GraphError(f"no node {id!r}") from None
|
|
57
|
+
|
|
58
|
+
def edge(self, id: str) -> Edge:
|
|
59
|
+
try:
|
|
60
|
+
return self._edge[id]
|
|
61
|
+
except KeyError:
|
|
62
|
+
raise GraphError(f"no edge {id!r}") from None
|
|
63
|
+
|
|
64
|
+
def __contains__(self, id) -> bool:
|
|
65
|
+
return id in self._node
|
|
66
|
+
|
|
67
|
+
def __len__(self) -> int:
|
|
68
|
+
return len(self.nodes)
|
|
69
|
+
|
|
70
|
+
def outgoing(self, node: str, port: str | None = None) -> tuple[Edge, ...]:
|
|
71
|
+
"""Edges leaving a node, or one of its output ports."""
|
|
72
|
+
ports = self._out[node]
|
|
73
|
+
return ports.get(port, ()) if port is not None else tuple(e for edges in ports.values() for e in edges)
|
|
74
|
+
|
|
75
|
+
def incoming(self, node: str, port: str | None = None) -> tuple[Edge, ...]:
|
|
76
|
+
"""Edges arriving at a node, or one of its input ports."""
|
|
77
|
+
ports = self._in[node]
|
|
78
|
+
return ports.get(port, ()) if port is not None else tuple(e for edges in ports.values() for e in edges)
|
|
79
|
+
|
|
80
|
+
def needs(self, node: str) -> int:
|
|
81
|
+
"""How many required inputs must arrive before the node's first activation (0: it starts by itself)."""
|
|
82
|
+
return self._needs[node]
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def sources(self) -> tuple[str, ...]:
|
|
86
|
+
"""Nodes that wait for nothing."""
|
|
87
|
+
return tuple(id for id in self.order if not self._needs[id])
|
|
88
|
+
|
|
89
|
+
def describe(self) -> dict:
|
|
90
|
+
"""The definition as plain data, in a fixed order: equal graphs describe, and so hash, the same."""
|
|
91
|
+
return {"nodes": sorted((n.describe() for n in self.nodes), key=lambda d: d["id"]),
|
|
92
|
+
"edges": sorted((e.describe() for e in self.edges), key=lambda d: d["id"]),
|
|
93
|
+
"inputs": {name: str(end) for name, end in sorted(self.inputs.items())},
|
|
94
|
+
"outputs": {name: str(end) for name, end in sorted(self.outputs.items())}}
|
|
95
|
+
|
|
96
|
+
def __repr__(self):
|
|
97
|
+
return f"<Graph {len(self.nodes)} nodes, {len(self.edges)} edges, {self.digest[:12]}>"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class GraphBuilder:
|
|
101
|
+
def __init__(self):
|
|
102
|
+
self._nodes: dict[str, Node] = {}
|
|
103
|
+
self._edges: dict[str, Edge] = {}
|
|
104
|
+
self._inputs: dict[str, Endpoint] = {}
|
|
105
|
+
self._outputs: dict[str, Endpoint] = {}
|
|
106
|
+
|
|
107
|
+
@classmethod
|
|
108
|
+
def of(cls, graph: "Graph") -> "GraphBuilder":
|
|
109
|
+
"""A builder holding everything ``graph`` has: the way to make a changed graph from an existing one."""
|
|
110
|
+
builder = cls()
|
|
111
|
+
builder._nodes = {node.id: node for node in graph.nodes}
|
|
112
|
+
builder._edges = {edge.id: edge for edge in graph.edges}
|
|
113
|
+
builder._inputs, builder._outputs = dict(graph.inputs), dict(graph.outputs)
|
|
114
|
+
return builder
|
|
115
|
+
|
|
116
|
+
def remove(self, node: str) -> "GraphBuilder":
|
|
117
|
+
"""Take a node out, with every edge and graph port that touches it."""
|
|
118
|
+
if node not in self._nodes:
|
|
119
|
+
raise GraphError(f"no node {node!r}")
|
|
120
|
+
del self._nodes[node]
|
|
121
|
+
self._edges = {id: e for id, e in self._edges.items() if node not in (e.source.node, e.target.node)}
|
|
122
|
+
self._inputs = {name: end for name, end in self._inputs.items() if end.node != node}
|
|
123
|
+
self._outputs = {name: end for name, end in self._outputs.items() if end.node != node}
|
|
124
|
+
return self
|
|
125
|
+
|
|
126
|
+
def disconnect(self, edge: str) -> "GraphBuilder":
|
|
127
|
+
if edge not in self._edges:
|
|
128
|
+
raise GraphError(f"no edge {edge!r}")
|
|
129
|
+
del self._edges[edge]
|
|
130
|
+
return self
|
|
131
|
+
|
|
132
|
+
def add(self, *nodes: Node) -> "GraphBuilder":
|
|
133
|
+
for node in nodes:
|
|
134
|
+
if not isinstance(node, Node):
|
|
135
|
+
raise GraphError(f"not a node: {node!r}")
|
|
136
|
+
if node.id in self._nodes:
|
|
137
|
+
raise GraphError(f"duplicate node: {node.id}")
|
|
138
|
+
self._nodes[node.id] = node
|
|
139
|
+
return self
|
|
140
|
+
|
|
141
|
+
def connect(self, source, target, protocol: str, *, id: str | None = None, back: bool = False,
|
|
142
|
+
max_rounds: int | None = None) -> "GraphBuilder":
|
|
143
|
+
"""Join ``"node.port"`` to ``"node.port"``. The id defaults to ``"a.out>b.in"``."""
|
|
144
|
+
try:
|
|
145
|
+
source, target = Endpoint.of(source), Endpoint.of(target)
|
|
146
|
+
edge = Edge(id or f"{source}>{target}", source, target, protocol, back, max_rounds)
|
|
147
|
+
except EdgeError as exc:
|
|
148
|
+
raise GraphError(str(exc)) from None
|
|
149
|
+
if edge.id in self._edges:
|
|
150
|
+
raise GraphError(f"duplicate edge: {edge.id}")
|
|
151
|
+
self._edges[edge.id] = edge
|
|
152
|
+
return self
|
|
153
|
+
|
|
154
|
+
def input(self, name: str, target) -> "GraphBuilder":
|
|
155
|
+
"""Name an input port as an input of the graph itself: whoever runs the graph feeds it."""
|
|
156
|
+
return self._expose(self._inputs, name, target)
|
|
157
|
+
|
|
158
|
+
def output(self, name: str, source) -> "GraphBuilder":
|
|
159
|
+
"""Name an output port as an output of the graph itself."""
|
|
160
|
+
return self._expose(self._outputs, name, source)
|
|
161
|
+
|
|
162
|
+
def _expose(self, table, name, end) -> "GraphBuilder":
|
|
163
|
+
if not isinstance(name, str) or not name or name in table:
|
|
164
|
+
raise GraphError(f"invalid or duplicate graph port: {name!r}")
|
|
165
|
+
try:
|
|
166
|
+
table[name] = Endpoint.of(end)
|
|
167
|
+
except EdgeError as exc:
|
|
168
|
+
raise GraphError(str(exc)) from None
|
|
169
|
+
return self
|
|
170
|
+
|
|
171
|
+
# ---- compile ---------------------------------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
def compile(self, protocols: Protocols) -> Graph:
|
|
174
|
+
nodes, edges = tuple(self._nodes.values()), tuple(self._edges.values())
|
|
175
|
+
if not nodes:
|
|
176
|
+
raise GraphError("a graph needs at least one node")
|
|
177
|
+
out_index: dict[str, dict[str, list]] = {n.id: {port: [] for port in n.outputs} for n in nodes}
|
|
178
|
+
in_index: dict[str, dict[str, list]] = {n.id: {port: [] for port in n.inputs} for n in nodes}
|
|
179
|
+
fed: dict[Endpoint, int] = {} # forward feeds of each input port
|
|
180
|
+
called: set[Endpoint] = set() # input ports that are the target of a call
|
|
181
|
+
for edge in edges:
|
|
182
|
+
source, target = self._port(edge.source, "outputs", edge), self._port(edge.target, "inputs", edge)
|
|
183
|
+
protocol = protocols.get(edge.protocol)
|
|
184
|
+
if protocol is None:
|
|
185
|
+
raise GraphError(f"edge {edge.id}: unknown protocol {edge.protocol!r}")
|
|
186
|
+
try:
|
|
187
|
+
protocol.check(source, target)
|
|
188
|
+
except ProtocolError as exc:
|
|
189
|
+
raise GraphError(f"edge {edge.id}: {exc}") from None
|
|
190
|
+
out_index[edge.source.node][edge.source.port].append(edge)
|
|
191
|
+
in_index[edge.target.node][edge.target.port].append(edge)
|
|
192
|
+
if protocol.delivery is Delivery.CALL:
|
|
193
|
+
called.add(edge.target) # any number of callers may ask the same port
|
|
194
|
+
elif not edge.back:
|
|
195
|
+
fed[edge.target] = fed.get(edge.target, 0) + 1
|
|
196
|
+
for name, end in self._inputs.items():
|
|
197
|
+
self._port(end, "inputs", f"graph input {name}")
|
|
198
|
+
fed[end] = fed.get(end, 0) + 1
|
|
199
|
+
for name, end in self._outputs.items():
|
|
200
|
+
self._port(end, "outputs", f"graph output {name}")
|
|
201
|
+
needs = {}
|
|
202
|
+
for node in nodes:
|
|
203
|
+
count = 0
|
|
204
|
+
for port in node.inputs.values():
|
|
205
|
+
feeds = fed.get(Endpoint(node.id, port.name), 0)
|
|
206
|
+
if feeds > 1 and not port.many:
|
|
207
|
+
raise GraphError(f"{node.id}.{port.name} takes one feed, {feeds} are connected")
|
|
208
|
+
if port.required:
|
|
209
|
+
if not feeds and Endpoint(node.id, port.name) not in called:
|
|
210
|
+
raise GraphError(f"{node.id}.{port.name} is required and nothing feeds it")
|
|
211
|
+
count += 1
|
|
212
|
+
if node.firing is Firing.ANY:
|
|
213
|
+
if not any(fed.get(Endpoint(node.id, name)) or in_index[node.id][name] for name in node.inputs):
|
|
214
|
+
raise GraphError(f"{node.id} fires on any input and nothing feeds it")
|
|
215
|
+
count = 1 # one arrival, on whichever port, is enough
|
|
216
|
+
needs[node.id] = count
|
|
217
|
+
order = self._order(nodes, edges)
|
|
218
|
+
self._check_loops(edges, out_index)
|
|
219
|
+
freeze = lambda index: {n: MappingProxyType({p: tuple(es) for p, es in ports.items()}) # noqa: E731
|
|
220
|
+
for n, ports in index.items()}
|
|
221
|
+
return Graph(nodes, edges, dict(self._inputs), dict(self._outputs), order, freeze(out_index), freeze(in_index), needs)
|
|
222
|
+
|
|
223
|
+
def _port(self, end: Endpoint, side: str, where):
|
|
224
|
+
node = self._nodes.get(end.node)
|
|
225
|
+
if node is None:
|
|
226
|
+
raise GraphError(f"{where if isinstance(where, str) else 'edge ' + where.id}: no node {end.node!r}")
|
|
227
|
+
port = getattr(node, side).get(end.port)
|
|
228
|
+
if port is None:
|
|
229
|
+
kind = "output" if side == "outputs" else "input"
|
|
230
|
+
raise GraphError(f"{where if isinstance(where, str) else 'edge ' + where.id}: {end.node} has no {kind} port {end.port!r}")
|
|
231
|
+
return port
|
|
232
|
+
|
|
233
|
+
@staticmethod
|
|
234
|
+
def _order(nodes: Iterable[Node], edges: Iterable[Edge]) -> tuple[str, ...]:
|
|
235
|
+
"""Upstream first, ignoring back edges; ties keep the order the nodes were added in."""
|
|
236
|
+
waiting = {n.id: 0 for n in nodes}
|
|
237
|
+
after: dict[str, list[str]] = {n.id: [] for n in nodes}
|
|
238
|
+
for edge in edges:
|
|
239
|
+
if not edge.back:
|
|
240
|
+
waiting[edge.target.node] += 1
|
|
241
|
+
after[edge.source.node].append(edge.target.node)
|
|
242
|
+
queue = deque(id for id, count in waiting.items() if not count)
|
|
243
|
+
order = []
|
|
244
|
+
while queue:
|
|
245
|
+
current = queue.popleft()
|
|
246
|
+
order.append(current)
|
|
247
|
+
for following in after[current]:
|
|
248
|
+
waiting[following] -= 1
|
|
249
|
+
if not waiting[following]:
|
|
250
|
+
queue.append(following)
|
|
251
|
+
if len(order) != len(waiting):
|
|
252
|
+
stuck = sorted(id for id, count in waiting.items() if count)
|
|
253
|
+
raise GraphError(f"a loop needs a back edge: {', '.join(stuck[:8])} form a cycle of forward edges")
|
|
254
|
+
return tuple(order)
|
|
255
|
+
|
|
256
|
+
@staticmethod
|
|
257
|
+
def _check_loops(edges: Iterable[Edge], out_index) -> None:
|
|
258
|
+
"""A back edge from A to B closes a loop only if B leads to A along forward edges."""
|
|
259
|
+
for edge in edges:
|
|
260
|
+
if not edge.back:
|
|
261
|
+
continue
|
|
262
|
+
goal, seen, stack = edge.source.node, {edge.target.node}, [edge.target.node]
|
|
263
|
+
while stack and goal not in seen:
|
|
264
|
+
for following in out_index[stack.pop()].values():
|
|
265
|
+
for forward in following:
|
|
266
|
+
if not forward.back and forward.target.node not in seen:
|
|
267
|
+
seen.add(forward.target.node)
|
|
268
|
+
stack.append(forward.target.node)
|
|
269
|
+
if goal not in seen:
|
|
270
|
+
raise GraphError(f"edge {edge.id} is marked back but closes no loop: "
|
|
271
|
+
f"{edge.target.node} does not lead to {edge.source.node}")
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""The method: a reusable way of joining nodes, a chain, a loop, a tree and so on.
|
|
2
|
+
|
|
3
|
+
A method takes the parts it is given and returns a graph. How the result runs is already in that graph: each
|
|
4
|
+
node says when it fires, each back edge says how many rounds it may open. A graph can be wrapped as one node,
|
|
5
|
+
so methods nest.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from abc import ABC, abstractmethod
|
|
10
|
+
from typing import ClassVar
|
|
11
|
+
|
|
12
|
+
from .graph import Graph
|
|
13
|
+
from .protocol import Protocols
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Method(ABC):
|
|
17
|
+
name: ClassVar[str] = ""
|
|
18
|
+
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def build(self, protocols: Protocols) -> Graph:
|
|
21
|
+
"""The graph this method makes of its parts."""
|
interlacing/base/node.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""The node: anything that takes information in and gives information out.
|
|
2
|
+
|
|
3
|
+
A node knows its ports and how to run once. It does not know which graph it is in, who is upstream or
|
|
4
|
+
downstream, or how its outputs travel: everything it touches outside itself goes through the ``Context`` the
|
|
5
|
+
engine hands it. An agent, a function, an external program, a device, a person and a whole sub-graph are all
|
|
6
|
+
nodes, with no difference in standing.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from enum import Enum
|
|
13
|
+
from types import MappingProxyType
|
|
14
|
+
from typing import Any, ClassVar, Iterable, Mapping
|
|
15
|
+
|
|
16
|
+
from .port import Port
|
|
17
|
+
|
|
18
|
+
ID = re.compile(r"[a-z][a-z0-9_-]{0,63}")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class NodeError(ValueError):
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Invalid(NodeError):
|
|
26
|
+
"""What a node sent, or was sent, did not pass a check. ``problem`` says what is wrong, in words meant
|
|
27
|
+
to be read by whoever can put it right. ``again`` is true when the node may correct it and send again
|
|
28
|
+
within the same activation; when it is false the node fails."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, port: str, problem: str, again: bool = False):
|
|
31
|
+
super().__init__(f"{port}: {problem}")
|
|
32
|
+
self.port, self.problem, self.again = port, problem, again
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Firing(Enum):
|
|
36
|
+
"""When a node is activated."""
|
|
37
|
+
|
|
38
|
+
ALL = "all" # once every required input has arrived
|
|
39
|
+
ANY = "any" # each time any input arrives
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Context(ABC):
|
|
43
|
+
"""What the engine gives a node for one activation."""
|
|
44
|
+
|
|
45
|
+
node: str # the node's id
|
|
46
|
+
round: int # which activation of this node this is, from 1
|
|
47
|
+
problems: list # when the activation is run again because what it sent was not accepted: what was wrong
|
|
48
|
+
within: Any # when the graph runs inside a node of another: (node id, round) of each, outermost first
|
|
49
|
+
key: Any # names this activation of this node in this run, and is the same when it runs again after
|
|
50
|
+
# an interruption: give it to an outside service so that it does not do the same thing twice
|
|
51
|
+
|
|
52
|
+
@abstractmethod
|
|
53
|
+
def emit(self, port: str, payload: Any) -> None:
|
|
54
|
+
"""Send ``payload`` out of an output port now (a node may emit many times during one activation).
|
|
55
|
+
Raises ``Invalid`` when the payload does not pass the port's checks; a node that can put it right
|
|
56
|
+
catches that, reads ``problem`` and, when ``again`` is true, sends a corrected payload."""
|
|
57
|
+
|
|
58
|
+
@abstractmethod
|
|
59
|
+
def call(self, port: str, payload: Any) -> Any:
|
|
60
|
+
"""Send ``payload`` out of an output port whose edge is a call, wait, and return the other node's answer."""
|
|
61
|
+
|
|
62
|
+
@abstractmethod
|
|
63
|
+
def stopping(self) -> bool:
|
|
64
|
+
"""True once the run is being stopped: a long activation should return soon."""
|
|
65
|
+
|
|
66
|
+
@abstractmethod
|
|
67
|
+
def engine(self, graph: Any, protocols: Any, *, workers: int = 0, firm: bool = False) -> Any:
|
|
68
|
+
"""An engine for a graph that runs inside this activation. It stops when the outer run stops, keeps
|
|
69
|
+
its own journal beside the outer one when there is one, tells the outer observer what happens in it,
|
|
70
|
+
and works under the same time limit. ``go(inputs)`` runs it, or takes it up where an interrupted
|
|
71
|
+
earlier attempt at this same activation left it. ``firm=True`` is for a graph that is not trusted:
|
|
72
|
+
none of its nodes may give itself more time than the run allows."""
|
|
73
|
+
|
|
74
|
+
def use(self, **amounts: float) -> None:
|
|
75
|
+
"""Report what this activation used up, under names of the node's choosing (``tokens=1200``,
|
|
76
|
+
``calls=1``, ``seconds=3.5``). The run adds it up; when a total goes over the run's budget for that
|
|
77
|
+
name the run fails, and this raises so that the node stops at once."""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _ports(given: Iterable[Port] | Mapping[str, Port] | None, declared: Iterable[Port] | Mapping[str, Port]) -> Mapping[str, Port]:
|
|
81
|
+
ports = list(given.values() if isinstance(given, Mapping) else given) if given is not None else list(declared)
|
|
82
|
+
out: dict[str, Port] = {}
|
|
83
|
+
for port in ports:
|
|
84
|
+
if not isinstance(port, Port):
|
|
85
|
+
raise NodeError(f"not a port: {port!r}")
|
|
86
|
+
if port.name in out:
|
|
87
|
+
raise NodeError(f"duplicate port: {port.name}")
|
|
88
|
+
out[port.name] = port
|
|
89
|
+
return MappingProxyType(out)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class Node(ABC):
|
|
93
|
+
"""Subclass, set ``kind``, declare ports (on the class or per instance) and implement ``run``.
|
|
94
|
+
|
|
95
|
+
``run`` receives the inputs that have arrived, by port name (a list for a ``many`` port). It may return a
|
|
96
|
+
mapping of output port to payload, call ``ctx.emit`` any number of times, or both. Returning ends the
|
|
97
|
+
activation.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
kind: ClassVar[str] = "" # the name this kind of node is registered and stored under
|
|
101
|
+
# A class may declare its ports as a tuple of Ports; on an instance they are a mapping from name to Port.
|
|
102
|
+
inputs: Mapping[str, Port] = () # type: ignore[assignment]
|
|
103
|
+
outputs: Mapping[str, Port] = () # type: ignore[assignment]
|
|
104
|
+
firing: Firing = Firing.ALL
|
|
105
|
+
repairs: int = 0 # how often an output may be corrected, for ports that do not say
|
|
106
|
+
time_limit: float | None = None # the longest one activation may take, in seconds; None: the engine's
|
|
107
|
+
|
|
108
|
+
def __init__(self, id: str, *, inputs: Iterable[Port] | None = None, outputs: Iterable[Port] | None = None,
|
|
109
|
+
firing: Firing | None = None):
|
|
110
|
+
if not isinstance(id, str) or not ID.fullmatch(id):
|
|
111
|
+
raise NodeError(f"invalid node id: {id!r}")
|
|
112
|
+
if not type(self).kind:
|
|
113
|
+
raise NodeError(f"{type(self).__name__} has no kind")
|
|
114
|
+
self.id = id
|
|
115
|
+
self.inputs = _ports(inputs, type(self).inputs)
|
|
116
|
+
self.outputs = _ports(outputs, type(self).outputs)
|
|
117
|
+
if firing is not None:
|
|
118
|
+
self.firing = Firing(firing)
|
|
119
|
+
|
|
120
|
+
def config(self) -> Mapping:
|
|
121
|
+
"""Plain data that sets this node apart from another of the same kind (it goes into the graph digest)."""
|
|
122
|
+
return {}
|
|
123
|
+
|
|
124
|
+
def describe(self) -> dict:
|
|
125
|
+
own = self.__dict__.get("time_limit")
|
|
126
|
+
return {"id": self.id, "kind": type(self).kind, "firing": self.firing.value,
|
|
127
|
+
"inputs": [p.describe() for p in self.inputs.values()],
|
|
128
|
+
"outputs": [p.describe() for p in self.outputs.values()], "config": dict(self.config()),
|
|
129
|
+
**({"time_limit": own} if own is not None else {})}
|
|
130
|
+
|
|
131
|
+
# ---- lifecycle: only run is required --------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
def open(self, ctx: Context) -> None:
|
|
134
|
+
"""Acquire what the node needs, once, before its first activation."""
|
|
135
|
+
|
|
136
|
+
@abstractmethod
|
|
137
|
+
def run(self, ctx: Context, inputs: Mapping[str, Any]) -> Mapping[str, Any] | None:
|
|
138
|
+
"""One activation."""
|
|
139
|
+
|
|
140
|
+
def close(self, ctx: Context) -> None:
|
|
141
|
+
"""Release what ``open`` acquired."""
|
|
142
|
+
|
|
143
|
+
def snapshot(self) -> Mapping:
|
|
144
|
+
"""Plain data from which ``restore`` can rebuild the node's own state after a restart."""
|
|
145
|
+
return {}
|
|
146
|
+
|
|
147
|
+
def restore(self, state: Mapping) -> None:
|
|
148
|
+
pass
|
|
149
|
+
|
|
150
|
+
def __repr__(self):
|
|
151
|
+
return f"<{type(self).__name__} {self.id}>"
|