without-dag 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- without_dag-0.0.1/PKG-INFO +59 -0
- without_dag-0.0.1/README.md +39 -0
- without_dag-0.0.1/pyproject.toml +31 -0
- without_dag-0.0.1/src/without_dag/__init__.py +19 -0
- without_dag-0.0.1/src/without_dag/execution.py +148 -0
- without_dag-0.0.1/src/without_dag/graph.py +438 -0
- without_dag-0.0.1/src/without_dag/py.typed +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: without-dag
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Bounded-concurrency execution of DAG-shaped async workflows for without, liftable into a Processor.
|
|
5
|
+
Author: Josh Karpel
|
|
6
|
+
Author-email: Josh Karpel <josh.karpel@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
9
|
+
Classifier: Framework :: AsyncIO
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Dist: without-core==0.0.1
|
|
18
|
+
Requires-Python: >=3.14
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# without-dag
|
|
22
|
+
|
|
23
|
+
Concurrent execution of DAG-shaped async workflows, liftable into a `without`
|
|
24
|
+
`Processor`. A `Processor[In, Out]` is otherwise an opaque stream-to-stream
|
|
25
|
+
closure; this package lets the *inside* of a per-event step be a graph of async
|
|
26
|
+
sub-steps that run with bounded concurrency and recombine into one output. It is
|
|
27
|
+
the value-level fan-out/fan-in the substrate leaves room for: one input value
|
|
28
|
+
drives many concurrent computations.
|
|
29
|
+
|
|
30
|
+
`Graph` is the typed frontend: a builder that threads value types through the
|
|
31
|
+
wiring so a mismatched dependency is a mypy error and a cycle is unrepresentable.
|
|
32
|
+
It compiles once into a `CompiledGraph`, an async callable reused per event:
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from without_dag import Graph
|
|
36
|
+
|
|
37
|
+
async def fetch(request: Request) -> Fetched: ...
|
|
38
|
+
async def parse(fetched: Fetched) -> Parsed: ...
|
|
39
|
+
async def render(fetched: Fetched, parsed: Parsed) -> Report: ...
|
|
40
|
+
|
|
41
|
+
graph, (request,) = Graph.of(Request)
|
|
42
|
+
fetched = graph.node(fetch, request)
|
|
43
|
+
parsed = graph.node(parse, fetched)
|
|
44
|
+
report = graph.node(render, fetched, parsed)
|
|
45
|
+
run = graph.build(output=report, limit=4)
|
|
46
|
+
|
|
47
|
+
result: Report = await run(some_request)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Each node runs once (memoized, so a diamond's shared ancestor executes a single
|
|
51
|
+
time with no glitch), acyclicity is proven at the boundary via stdlib
|
|
52
|
+
`graphlib`, and a single-input graph is an async `(In) -> Out` callable, exactly
|
|
53
|
+
what `from_map` lifts into a `Processor`.
|
|
54
|
+
|
|
55
|
+
See the
|
|
56
|
+
[`without-dag` guide](https://without.help/guides/without-dag/)
|
|
57
|
+
(with the [API reference](https://without.help/reference/without_dag/))
|
|
58
|
+
for the full surface: the object-seam execution core (`Plan`, `drive`,
|
|
59
|
+
`evaluate`), the typed frontend, and lifting into a processor.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# without-dag
|
|
2
|
+
|
|
3
|
+
Concurrent execution of DAG-shaped async workflows, liftable into a `without`
|
|
4
|
+
`Processor`. A `Processor[In, Out]` is otherwise an opaque stream-to-stream
|
|
5
|
+
closure; this package lets the *inside* of a per-event step be a graph of async
|
|
6
|
+
sub-steps that run with bounded concurrency and recombine into one output. It is
|
|
7
|
+
the value-level fan-out/fan-in the substrate leaves room for: one input value
|
|
8
|
+
drives many concurrent computations.
|
|
9
|
+
|
|
10
|
+
`Graph` is the typed frontend: a builder that threads value types through the
|
|
11
|
+
wiring so a mismatched dependency is a mypy error and a cycle is unrepresentable.
|
|
12
|
+
It compiles once into a `CompiledGraph`, an async callable reused per event:
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from without_dag import Graph
|
|
16
|
+
|
|
17
|
+
async def fetch(request: Request) -> Fetched: ...
|
|
18
|
+
async def parse(fetched: Fetched) -> Parsed: ...
|
|
19
|
+
async def render(fetched: Fetched, parsed: Parsed) -> Report: ...
|
|
20
|
+
|
|
21
|
+
graph, (request,) = Graph.of(Request)
|
|
22
|
+
fetched = graph.node(fetch, request)
|
|
23
|
+
parsed = graph.node(parse, fetched)
|
|
24
|
+
report = graph.node(render, fetched, parsed)
|
|
25
|
+
run = graph.build(output=report, limit=4)
|
|
26
|
+
|
|
27
|
+
result: Report = await run(some_request)
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Each node runs once (memoized, so a diamond's shared ancestor executes a single
|
|
31
|
+
time with no glitch), acyclicity is proven at the boundary via stdlib
|
|
32
|
+
`graphlib`, and a single-input graph is an async `(In) -> Out` callable, exactly
|
|
33
|
+
what `from_map` lifts into a `Processor`.
|
|
34
|
+
|
|
35
|
+
See the
|
|
36
|
+
[`without-dag` guide](https://without.help/guides/without-dag/)
|
|
37
|
+
(with the [API reference](https://without.help/reference/without_dag/))
|
|
38
|
+
for the full surface: the object-seam execution core (`Plan`, `drive`,
|
|
39
|
+
`evaluate`), the typed frontend, and lifting into a processor.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.11.25,<0.12"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "without-dag"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
description = "Bounded-concurrency execution of DAG-shaped async workflows for without, liftable into a Processor."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Josh Karpel", email = "josh.karpel@gmail.com" },
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.14"
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
17
|
+
"Framework :: AsyncIO",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Programming Language :: Python :: 3.14",
|
|
23
|
+
"Topic :: Software Development :: Libraries",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"without-core==0.0.1",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[tool.uv.sources]
|
|
31
|
+
without-core = { workspace = true }
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from without_dag.execution import Node
|
|
2
|
+
from without_dag.execution import NodeKey
|
|
3
|
+
from without_dag.execution import Plan
|
|
4
|
+
from without_dag.execution import drive
|
|
5
|
+
from without_dag.execution import evaluate
|
|
6
|
+
from without_dag.graph import CompiledGraph
|
|
7
|
+
from without_dag.graph import Graph
|
|
8
|
+
from without_dag.graph import Handle
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"CompiledGraph",
|
|
12
|
+
"Graph",
|
|
13
|
+
"Handle",
|
|
14
|
+
"Node",
|
|
15
|
+
"NodeKey",
|
|
16
|
+
"Plan",
|
|
17
|
+
"drive",
|
|
18
|
+
"evaluate",
|
|
19
|
+
]
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from collections import Counter
|
|
5
|
+
from collections import deque
|
|
6
|
+
from collections.abc import AsyncGenerator
|
|
7
|
+
from collections.abc import Awaitable
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from collections.abc import Hashable
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from graphlib import TopologicalSorter
|
|
14
|
+
|
|
15
|
+
from without import cancel_futures
|
|
16
|
+
|
|
17
|
+
type NodeKey = Hashable
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, slots=True)
|
|
21
|
+
class Node:
|
|
22
|
+
"""
|
|
23
|
+
One async step in a graph, named by `key` and wired by `dependencies`.
|
|
24
|
+
|
|
25
|
+
The narrow seam a graph-defining frontend lowers onto: a `Node` is a value,
|
|
26
|
+
not a place. `run` receives its dependencies' results as a tuple in
|
|
27
|
+
`dependencies` order and returns this node's single result. Results cross
|
|
28
|
+
this seam as `object` (the executor cannot know each step's type); a typed
|
|
29
|
+
frontend restores precision above it, exactly as `without-web`'s `Extractor`
|
|
30
|
+
is collected as `Extractor[object]` and re-typed by `into`.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
key: NodeKey
|
|
34
|
+
dependencies: tuple[NodeKey, ...]
|
|
35
|
+
run: Callable[[tuple[object, ...]], Awaitable[object]]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True, slots=True)
|
|
39
|
+
class Plan:
|
|
40
|
+
"""
|
|
41
|
+
A node set compiled into its input-independent scheduling structure.
|
|
42
|
+
|
|
43
|
+
Everything a run needs that does *not* depend on the input values: the nodes
|
|
44
|
+
by key, the dependency edges (as the graph `graphlib` wants), and the number
|
|
45
|
+
of consumers per key (so a result can be freed once its last dependent has
|
|
46
|
+
read it). Computed once and reused across runs as a value, so a fixed graph
|
|
47
|
+
driven per event never rebuilds any of it.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
by_key: Mapping[NodeKey, Node]
|
|
51
|
+
dependencies: Mapping[NodeKey, tuple[NodeKey, ...]]
|
|
52
|
+
consumers: Mapping[NodeKey, int]
|
|
53
|
+
|
|
54
|
+
@classmethod
|
|
55
|
+
def of(cls, nodes: Iterable[Node]) -> Plan:
|
|
56
|
+
"""Compile a node set into a reusable `Plan`."""
|
|
57
|
+
by_key = {node.key: node for node in nodes}
|
|
58
|
+
dependencies = {key: node.dependencies for key, node in by_key.items()}
|
|
59
|
+
consumers: Counter[NodeKey] = Counter()
|
|
60
|
+
for node in by_key.values():
|
|
61
|
+
consumers.update(node.dependencies)
|
|
62
|
+
return cls(by_key=by_key, dependencies=dependencies, consumers=consumers)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def drive(
|
|
66
|
+
plan: Plan,
|
|
67
|
+
inputs: Mapping[NodeKey, object],
|
|
68
|
+
limit: int | None,
|
|
69
|
+
) -> AsyncGenerator[tuple[NodeKey, object]]:
|
|
70
|
+
"""
|
|
71
|
+
Run a compiled `Plan`, yielding each `(key, result)` the instant it completes.
|
|
72
|
+
|
|
73
|
+
The streaming core, and the *events* half of the model. Yields completions in
|
|
74
|
+
whatever order nodes finish; the only ordering guarantee is the causal one, a
|
|
75
|
+
node after the dependencies it consumed. `inputs` pre-supplies the values of
|
|
76
|
+
source keys, marked done without running and never yielded. `limit` caps how
|
|
77
|
+
many nodes run concurrently (`None` is unbounded).
|
|
78
|
+
|
|
79
|
+
Scheduling replicates the shape of `without.limit_concurrency`
|
|
80
|
+
(`asyncio.wait(..., return_when=FIRST_COMPLETED)`) rather than calling it: the
|
|
81
|
+
scheduler needs the completed task's `NodeKey` to unlock successors, which
|
|
82
|
+
that lazy source hides. Acyclicity is proven by `TopologicalSorter.prepare`,
|
|
83
|
+
which raises `graphlib.CycleError`. Each node runs once; a result is dropped
|
|
84
|
+
as soon as its last dependent has captured it. A node that raises fails the
|
|
85
|
+
whole run, cancelling in-flight siblings, which is also how closing the
|
|
86
|
+
iterator early tears the run down.
|
|
87
|
+
"""
|
|
88
|
+
if limit is not None and limit < 1:
|
|
89
|
+
raise ValueError(f"limit must be at least 1 or None, but got {limit}")
|
|
90
|
+
|
|
91
|
+
sorter: TopologicalSorter[NodeKey] = TopologicalSorter(plan.dependencies)
|
|
92
|
+
sorter.prepare()
|
|
93
|
+
consumers: Counter[NodeKey] = Counter(plan.consumers)
|
|
94
|
+
results: dict[NodeKey, object] = dict(inputs)
|
|
95
|
+
running: dict[asyncio.Future[object], NodeKey] = {}
|
|
96
|
+
ready: deque[NodeKey] = deque(sorter.get_ready())
|
|
97
|
+
try:
|
|
98
|
+
while sorter.is_active():
|
|
99
|
+
while ready and (limit is None or len(running) < limit):
|
|
100
|
+
key = ready.popleft()
|
|
101
|
+
if key in results:
|
|
102
|
+
sorter.done(key)
|
|
103
|
+
ready.extend(sorter.get_ready())
|
|
104
|
+
continue
|
|
105
|
+
if key not in plan.by_key:
|
|
106
|
+
raise KeyError(f"{key!r} is neither a supplied input nor a defined node")
|
|
107
|
+
node = plan.by_key[key]
|
|
108
|
+
args = tuple(results[dependency] for dependency in node.dependencies)
|
|
109
|
+
for dependency in node.dependencies:
|
|
110
|
+
consumers[dependency] -= 1
|
|
111
|
+
if consumers[dependency] == 0:
|
|
112
|
+
del results[dependency]
|
|
113
|
+
running[asyncio.ensure_future(node.run(args))] = key
|
|
114
|
+
done, _ = await asyncio.wait(running, return_when=asyncio.FIRST_COMPLETED)
|
|
115
|
+
for future in done:
|
|
116
|
+
key = running.pop(future)
|
|
117
|
+
value = future.result()
|
|
118
|
+
results[key] = value
|
|
119
|
+
sorter.done(key)
|
|
120
|
+
ready.extend(sorter.get_ready())
|
|
121
|
+
yield key, value
|
|
122
|
+
finally:
|
|
123
|
+
await cancel_futures(running)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
async def evaluate(plan: Plan, target: NodeKey, inputs: Mapping[NodeKey, object], limit: int | None) -> object:
|
|
127
|
+
"""
|
|
128
|
+
Run every node in `plan` and return `target`'s value: the *behavior* read.
|
|
129
|
+
|
|
130
|
+
A consumer of `drive` that runs the whole graph and keeps the one value the
|
|
131
|
+
caller wants, dropping the rest. `target` is a node whose completion supplies
|
|
132
|
+
the value, or a supplied input returned directly (an identity plan). There is
|
|
133
|
+
deliberately no early return on `target`: the graph is run to completion, so
|
|
134
|
+
the result reflects the whole graph and every node's effects have happened.
|
|
135
|
+
|
|
136
|
+
A `target` that is neither a defined node nor a supplied input raises
|
|
137
|
+
`KeyError`, matching `drive`, rather than silently reading back as `None`.
|
|
138
|
+
"""
|
|
139
|
+
if target not in plan.by_key and target not in inputs:
|
|
140
|
+
raise KeyError(f"{target!r} is neither a supplied input nor a defined node")
|
|
141
|
+
# `target` is either a node, whose completion `drive` yields (and overwrites
|
|
142
|
+
# this), or a supplied input, which `drive` never yields: default to its value
|
|
143
|
+
# so an identity graph (the output is an input) returns it.
|
|
144
|
+
result: object = inputs.get(target)
|
|
145
|
+
async for key, value in drive(plan, inputs, limit):
|
|
146
|
+
if key == target:
|
|
147
|
+
result = value
|
|
148
|
+
return result
|
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncGenerator
|
|
4
|
+
from collections.abc import Awaitable
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from dataclasses import field
|
|
8
|
+
from typing import Generic
|
|
9
|
+
from typing import TypeVar
|
|
10
|
+
from typing import cast
|
|
11
|
+
from typing import overload
|
|
12
|
+
|
|
13
|
+
from without_dag.execution import Node
|
|
14
|
+
from without_dag.execution import NodeKey
|
|
15
|
+
from without_dag.execution import Plan
|
|
16
|
+
from without_dag.execution import drive
|
|
17
|
+
from without_dag.execution import evaluate
|
|
18
|
+
|
|
19
|
+
# Covariant: `T` names the node's result but is a phantom (no field holds it), so
|
|
20
|
+
# a `Handle[int]` is a `Handle[object]`. That is what lets the `node` runtime
|
|
21
|
+
# collect a heterogeneous mix of dependency handles as `*deps: Handle[object]`
|
|
22
|
+
# while the overloads keep each one's precise type. The legacy `TypeVar` is
|
|
23
|
+
# needed because PEP 695 infers an unused parameter as invariant; the variance is
|
|
24
|
+
# sound here, so we state it explicitly (see `without_web.extractors.Extractor`).
|
|
25
|
+
_T_co = TypeVar("_T_co", covariant=True)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True, slots=True)
|
|
29
|
+
class Handle(Generic[_T_co]): # noqa: UP046 - PEP 695 infers a phantom parameter as invariant; the covariant TypeVar is deliberate (see above)
|
|
30
|
+
"""
|
|
31
|
+
A typed reference to a node's future result.
|
|
32
|
+
|
|
33
|
+
The token the builder hands back from `of`/`node` and takes back as a
|
|
34
|
+
dependency. `T` is phantom: the handle carries only an opaque `key`, but the
|
|
35
|
+
type flows through the wiring so a downstream step is checked against the
|
|
36
|
+
types of the handles it depends on. Because a caller can only pass handles
|
|
37
|
+
that already exist, a cycle is unrepresentable through this API.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
key: NodeKey
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class CompiledGraph[*Ins, Out]:
|
|
45
|
+
"""
|
|
46
|
+
A frozen graph that *is* an async callable `(*Ins) -> Out`.
|
|
47
|
+
|
|
48
|
+
`build` returns one of these. It runs one bounded-concurrency execution per
|
|
49
|
+
call, seeding each entry the graph was opened over with the matching
|
|
50
|
+
positional argument and returning the value of its `output`. A single-input
|
|
51
|
+
graph is a plain `Callable[[In], Awaitable[Out]]`, so it lifts into a
|
|
52
|
+
`Processor` with `from_map`. The scheduling structure is compiled once at
|
|
53
|
+
`build` into `_plan` and reused by every call and `stream`, so a graph driven
|
|
54
|
+
per event never recomputes it. `nodes` is kept so the structure is
|
|
55
|
+
*recoverable* (a future diagram is derived from this one declaration, not
|
|
56
|
+
maintained beside it).
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
nodes: tuple[Node, ...]
|
|
60
|
+
inputs: tuple[NodeKey, ...]
|
|
61
|
+
output: NodeKey
|
|
62
|
+
limit: int | None
|
|
63
|
+
_plan: Plan
|
|
64
|
+
|
|
65
|
+
async def __call__(self, *values: *Ins) -> Out:
|
|
66
|
+
result = await evaluate(self._plan, self.output, self._seed(values), self.limit)
|
|
67
|
+
return cast(Out, result)
|
|
68
|
+
|
|
69
|
+
def stream(self, *values: *Ins) -> AsyncGenerator[tuple[NodeKey, object]]:
|
|
70
|
+
"""
|
|
71
|
+
Run the whole graph, yielding each node's `(key, result)` as it completes.
|
|
72
|
+
|
|
73
|
+
The streaming counterpart to calling the graph: `__call__` samples the
|
|
74
|
+
single `output` value (a behavior), `stream` reports every completion as
|
|
75
|
+
it happens (the events), letting the caller react as results land or read
|
|
76
|
+
several outputs. The positional arguments seed the graph's inputs,
|
|
77
|
+
checked against `*Ins` exactly as the call is; match a yielded key against
|
|
78
|
+
a `Handle`'s `key` to pick out a node's result.
|
|
79
|
+
"""
|
|
80
|
+
return drive(self._plan, self._seed(values), self.limit)
|
|
81
|
+
|
|
82
|
+
def _seed(self, values: tuple[object, ...]) -> dict[NodeKey, object]:
|
|
83
|
+
return dict(zip(self.inputs, values, strict=True))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True, slots=True)
|
|
87
|
+
class Graph[*Ins]:
|
|
88
|
+
"""
|
|
89
|
+
A builder that records async steps and returns typed `Handle`s.
|
|
90
|
+
|
|
91
|
+
`of` opens a graph over its entry types and hands back a tuple of one `Handle`
|
|
92
|
+
per type, `node` adds a step wired to the handles it depends on, and `build` freezes the
|
|
93
|
+
result into a `CompiledGraph`. Because the graph carries its entry pack in
|
|
94
|
+
its type (`Graph[*Ins]`), `build` needs only the output handle: it recovers
|
|
95
|
+
the inputs the graph already knows, so there is no second place to keep in
|
|
96
|
+
sync. The builder itself is a frozen value; the only mutation is appending to
|
|
97
|
+
its interior list of recorded nodes. Each step's function receives its
|
|
98
|
+
dependencies' results as positional arguments in the order its handles were
|
|
99
|
+
passed.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
_nodes: list[Node] = field(default_factory=list)
|
|
103
|
+
_input_keys: tuple[NodeKey, ...] = ()
|
|
104
|
+
|
|
105
|
+
# [[[cog import cog; from dag_ladders import emit; cog.outl(emit("of")) ]]]
|
|
106
|
+
@overload
|
|
107
|
+
@staticmethod
|
|
108
|
+
def of() -> tuple[
|
|
109
|
+
Graph[()],
|
|
110
|
+
tuple[()],
|
|
111
|
+
]: ...
|
|
112
|
+
|
|
113
|
+
@overload
|
|
114
|
+
@staticmethod
|
|
115
|
+
def of[A](
|
|
116
|
+
a: type[A],
|
|
117
|
+
/,
|
|
118
|
+
) -> tuple[
|
|
119
|
+
Graph[A],
|
|
120
|
+
tuple[Handle[A]],
|
|
121
|
+
]: ...
|
|
122
|
+
|
|
123
|
+
@overload
|
|
124
|
+
@staticmethod
|
|
125
|
+
def of[A, B](
|
|
126
|
+
a: type[A],
|
|
127
|
+
b: type[B],
|
|
128
|
+
/,
|
|
129
|
+
) -> tuple[
|
|
130
|
+
Graph[A, B],
|
|
131
|
+
tuple[Handle[A], Handle[B]],
|
|
132
|
+
]: ...
|
|
133
|
+
|
|
134
|
+
@overload
|
|
135
|
+
@staticmethod
|
|
136
|
+
def of[A, B, C](
|
|
137
|
+
a: type[A],
|
|
138
|
+
b: type[B],
|
|
139
|
+
c: type[C],
|
|
140
|
+
/,
|
|
141
|
+
) -> tuple[
|
|
142
|
+
Graph[A, B, C],
|
|
143
|
+
tuple[Handle[A], Handle[B], Handle[C]],
|
|
144
|
+
]: ...
|
|
145
|
+
|
|
146
|
+
@overload
|
|
147
|
+
@staticmethod
|
|
148
|
+
def of[A, B, C, D](
|
|
149
|
+
a: type[A],
|
|
150
|
+
b: type[B],
|
|
151
|
+
c: type[C],
|
|
152
|
+
d: type[D],
|
|
153
|
+
/,
|
|
154
|
+
) -> tuple[
|
|
155
|
+
Graph[A, B, C, D],
|
|
156
|
+
tuple[Handle[A], Handle[B], Handle[C], Handle[D]],
|
|
157
|
+
]: ...
|
|
158
|
+
|
|
159
|
+
@overload
|
|
160
|
+
@staticmethod
|
|
161
|
+
def of[A, B, C, D, E](
|
|
162
|
+
a: type[A],
|
|
163
|
+
b: type[B],
|
|
164
|
+
c: type[C],
|
|
165
|
+
d: type[D],
|
|
166
|
+
e: type[E],
|
|
167
|
+
/,
|
|
168
|
+
) -> tuple[
|
|
169
|
+
Graph[A, B, C, D, E],
|
|
170
|
+
tuple[Handle[A], Handle[B], Handle[C], Handle[D], Handle[E]],
|
|
171
|
+
]: ...
|
|
172
|
+
|
|
173
|
+
@overload
|
|
174
|
+
@staticmethod
|
|
175
|
+
def of[A, B, C, D, E, F](
|
|
176
|
+
a: type[A],
|
|
177
|
+
b: type[B],
|
|
178
|
+
c: type[C],
|
|
179
|
+
d: type[D],
|
|
180
|
+
e: type[E],
|
|
181
|
+
f: type[F],
|
|
182
|
+
/,
|
|
183
|
+
) -> tuple[
|
|
184
|
+
Graph[A, B, C, D, E, F],
|
|
185
|
+
tuple[Handle[A], Handle[B], Handle[C], Handle[D], Handle[E], Handle[F]],
|
|
186
|
+
]: ...
|
|
187
|
+
|
|
188
|
+
@overload
|
|
189
|
+
@staticmethod
|
|
190
|
+
def of[A, B, C, D, E, F, G](
|
|
191
|
+
a: type[A],
|
|
192
|
+
b: type[B],
|
|
193
|
+
c: type[C],
|
|
194
|
+
d: type[D],
|
|
195
|
+
e: type[E],
|
|
196
|
+
f: type[F],
|
|
197
|
+
g: type[G],
|
|
198
|
+
/,
|
|
199
|
+
) -> tuple[
|
|
200
|
+
Graph[A, B, C, D, E, F, G],
|
|
201
|
+
tuple[Handle[A], Handle[B], Handle[C], Handle[D], Handle[E], Handle[F], Handle[G]],
|
|
202
|
+
]: ...
|
|
203
|
+
|
|
204
|
+
@overload
|
|
205
|
+
@staticmethod
|
|
206
|
+
def of[A, B, C, D, E, F, G, H](
|
|
207
|
+
a: type[A],
|
|
208
|
+
b: type[B],
|
|
209
|
+
c: type[C],
|
|
210
|
+
d: type[D],
|
|
211
|
+
e: type[E],
|
|
212
|
+
f: type[F],
|
|
213
|
+
g: type[G],
|
|
214
|
+
h: type[H],
|
|
215
|
+
/,
|
|
216
|
+
) -> tuple[
|
|
217
|
+
Graph[A, B, C, D, E, F, G, H],
|
|
218
|
+
tuple[Handle[A], Handle[B], Handle[C], Handle[D], Handle[E], Handle[F], Handle[G], Handle[H]],
|
|
219
|
+
]: ...
|
|
220
|
+
|
|
221
|
+
@overload
|
|
222
|
+
@staticmethod
|
|
223
|
+
def of[A, B, C, D, E, F, G, H, J](
|
|
224
|
+
a: type[A],
|
|
225
|
+
b: type[B],
|
|
226
|
+
c: type[C],
|
|
227
|
+
d: type[D],
|
|
228
|
+
e: type[E],
|
|
229
|
+
f: type[F],
|
|
230
|
+
g: type[G],
|
|
231
|
+
h: type[H],
|
|
232
|
+
j: type[J],
|
|
233
|
+
/,
|
|
234
|
+
) -> tuple[
|
|
235
|
+
Graph[A, B, C, D, E, F, G, H, J],
|
|
236
|
+
tuple[Handle[A], Handle[B], Handle[C], Handle[D], Handle[E], Handle[F], Handle[G], Handle[H], Handle[J]],
|
|
237
|
+
]: ...
|
|
238
|
+
|
|
239
|
+
@overload
|
|
240
|
+
@staticmethod
|
|
241
|
+
def of[A, B, C, D, E, F, G, H, J, K](
|
|
242
|
+
a: type[A],
|
|
243
|
+
b: type[B],
|
|
244
|
+
c: type[C],
|
|
245
|
+
d: type[D],
|
|
246
|
+
e: type[E],
|
|
247
|
+
f: type[F],
|
|
248
|
+
g: type[G],
|
|
249
|
+
h: type[H],
|
|
250
|
+
j: type[J],
|
|
251
|
+
k: type[K],
|
|
252
|
+
/,
|
|
253
|
+
) -> tuple[
|
|
254
|
+
Graph[A, B, C, D, E, F, G, H, J, K],
|
|
255
|
+
tuple[
|
|
256
|
+
Handle[A], Handle[B], Handle[C], Handle[D], Handle[E], Handle[F], Handle[G], Handle[H], Handle[J], Handle[K]
|
|
257
|
+
],
|
|
258
|
+
]: ...
|
|
259
|
+
# [[[end]]]
|
|
260
|
+
@staticmethod
|
|
261
|
+
def of(*inputs: type[object]) -> tuple[object, ...]:
|
|
262
|
+
"""Open a graph over `inputs`, returning it and a tuple of one `Handle` per entry type."""
|
|
263
|
+
handles: tuple[Handle[object], ...] = tuple(Handle(object()) for _ in inputs)
|
|
264
|
+
graph: Graph[*tuple[object, ...]] = Graph(_input_keys=tuple(handle.key for handle in handles))
|
|
265
|
+
return (graph, handles)
|
|
266
|
+
|
|
267
|
+
# [[[cog cog.outl(emit("node")) ]]]
|
|
268
|
+
@overload
|
|
269
|
+
def node[T](
|
|
270
|
+
self,
|
|
271
|
+
fn: Callable[[], Awaitable[T]],
|
|
272
|
+
/,
|
|
273
|
+
) -> Handle[T]: ...
|
|
274
|
+
|
|
275
|
+
@overload
|
|
276
|
+
def node[T, A](
|
|
277
|
+
self,
|
|
278
|
+
fn: Callable[[A], Awaitable[T]],
|
|
279
|
+
a: Handle[A],
|
|
280
|
+
/,
|
|
281
|
+
) -> Handle[T]: ...
|
|
282
|
+
|
|
283
|
+
@overload
|
|
284
|
+
def node[T, A, B](
|
|
285
|
+
self,
|
|
286
|
+
fn: Callable[[A, B], Awaitable[T]],
|
|
287
|
+
a: Handle[A],
|
|
288
|
+
b: Handle[B],
|
|
289
|
+
/,
|
|
290
|
+
) -> Handle[T]: ...
|
|
291
|
+
|
|
292
|
+
@overload
|
|
293
|
+
def node[T, A, B, C](
|
|
294
|
+
self,
|
|
295
|
+
fn: Callable[[A, B, C], Awaitable[T]],
|
|
296
|
+
a: Handle[A],
|
|
297
|
+
b: Handle[B],
|
|
298
|
+
c: Handle[C],
|
|
299
|
+
/,
|
|
300
|
+
) -> Handle[T]: ...
|
|
301
|
+
|
|
302
|
+
@overload
|
|
303
|
+
def node[T, A, B, C, D](
|
|
304
|
+
self,
|
|
305
|
+
fn: Callable[[A, B, C, D], Awaitable[T]],
|
|
306
|
+
a: Handle[A],
|
|
307
|
+
b: Handle[B],
|
|
308
|
+
c: Handle[C],
|
|
309
|
+
d: Handle[D],
|
|
310
|
+
/,
|
|
311
|
+
) -> Handle[T]: ...
|
|
312
|
+
|
|
313
|
+
@overload
|
|
314
|
+
def node[T, A, B, C, D, E](
|
|
315
|
+
self,
|
|
316
|
+
fn: Callable[[A, B, C, D, E], Awaitable[T]],
|
|
317
|
+
a: Handle[A],
|
|
318
|
+
b: Handle[B],
|
|
319
|
+
c: Handle[C],
|
|
320
|
+
d: Handle[D],
|
|
321
|
+
e: Handle[E],
|
|
322
|
+
/,
|
|
323
|
+
) -> Handle[T]: ...
|
|
324
|
+
|
|
325
|
+
@overload
|
|
326
|
+
def node[T, A, B, C, D, E, F](
|
|
327
|
+
self,
|
|
328
|
+
fn: Callable[[A, B, C, D, E, F], Awaitable[T]],
|
|
329
|
+
a: Handle[A],
|
|
330
|
+
b: Handle[B],
|
|
331
|
+
c: Handle[C],
|
|
332
|
+
d: Handle[D],
|
|
333
|
+
e: Handle[E],
|
|
334
|
+
f: Handle[F],
|
|
335
|
+
/,
|
|
336
|
+
) -> Handle[T]: ...
|
|
337
|
+
|
|
338
|
+
@overload
|
|
339
|
+
def node[T, A, B, C, D, E, F, G](
|
|
340
|
+
self,
|
|
341
|
+
fn: Callable[[A, B, C, D, E, F, G], Awaitable[T]],
|
|
342
|
+
a: Handle[A],
|
|
343
|
+
b: Handle[B],
|
|
344
|
+
c: Handle[C],
|
|
345
|
+
d: Handle[D],
|
|
346
|
+
e: Handle[E],
|
|
347
|
+
f: Handle[F],
|
|
348
|
+
g: Handle[G],
|
|
349
|
+
/,
|
|
350
|
+
) -> Handle[T]: ...
|
|
351
|
+
|
|
352
|
+
@overload
|
|
353
|
+
def node[T, A, B, C, D, E, F, G, H](
|
|
354
|
+
self,
|
|
355
|
+
fn: Callable[[A, B, C, D, E, F, G, H], Awaitable[T]],
|
|
356
|
+
a: Handle[A],
|
|
357
|
+
b: Handle[B],
|
|
358
|
+
c: Handle[C],
|
|
359
|
+
d: Handle[D],
|
|
360
|
+
e: Handle[E],
|
|
361
|
+
f: Handle[F],
|
|
362
|
+
g: Handle[G],
|
|
363
|
+
h: Handle[H],
|
|
364
|
+
/,
|
|
365
|
+
) -> Handle[T]: ...
|
|
366
|
+
|
|
367
|
+
@overload
|
|
368
|
+
def node[T, A, B, C, D, E, F, G, H, J](
|
|
369
|
+
self,
|
|
370
|
+
fn: Callable[[A, B, C, D, E, F, G, H, J], Awaitable[T]],
|
|
371
|
+
a: Handle[A],
|
|
372
|
+
b: Handle[B],
|
|
373
|
+
c: Handle[C],
|
|
374
|
+
d: Handle[D],
|
|
375
|
+
e: Handle[E],
|
|
376
|
+
f: Handle[F],
|
|
377
|
+
g: Handle[G],
|
|
378
|
+
h: Handle[H],
|
|
379
|
+
j: Handle[J],
|
|
380
|
+
/,
|
|
381
|
+
) -> Handle[T]: ...
|
|
382
|
+
|
|
383
|
+
@overload
|
|
384
|
+
def node[T, A, B, C, D, E, F, G, H, J, K](
|
|
385
|
+
self,
|
|
386
|
+
fn: Callable[[A, B, C, D, E, F, G, H, J, K], Awaitable[T]],
|
|
387
|
+
a: Handle[A],
|
|
388
|
+
b: Handle[B],
|
|
389
|
+
c: Handle[C],
|
|
390
|
+
d: Handle[D],
|
|
391
|
+
e: Handle[E],
|
|
392
|
+
f: Handle[F],
|
|
393
|
+
g: Handle[G],
|
|
394
|
+
h: Handle[H],
|
|
395
|
+
j: Handle[J],
|
|
396
|
+
k: Handle[K],
|
|
397
|
+
/,
|
|
398
|
+
) -> Handle[T]: ...
|
|
399
|
+
# [[[end]]]
|
|
400
|
+
def node[T](self, fn: Callable[..., Awaitable[T]], *deps: Handle[object]) -> Handle[T]:
|
|
401
|
+
"""
|
|
402
|
+
Add a node computing `fn` from the handles it depends on, returning its
|
|
403
|
+
result handle.
|
|
404
|
+
|
|
405
|
+
`fn` is called with the dependencies' results as positional arguments in
|
|
406
|
+
the order their handles are passed. The overloads above tie each handle's
|
|
407
|
+
type to `fn`'s matching parameter, so a mismatch is a static error.
|
|
408
|
+
"""
|
|
409
|
+
key = object()
|
|
410
|
+
dependencies = tuple(dep.key for dep in deps)
|
|
411
|
+
|
|
412
|
+
async def run(args: tuple[object, ...]) -> object:
|
|
413
|
+
return await fn(*args)
|
|
414
|
+
|
|
415
|
+
self._nodes.append(Node(key=key, dependencies=dependencies, run=run))
|
|
416
|
+
return Handle(key)
|
|
417
|
+
|
|
418
|
+
def build[Out](self, *, output: Handle[Out], limit: int | None = None) -> CompiledGraph[*Ins, Out]:
|
|
419
|
+
"""
|
|
420
|
+
Freeze the recorded steps into a callable graph over the graph's inputs.
|
|
421
|
+
|
|
422
|
+
The scheduling structure is compiled once here, so running the graph
|
|
423
|
+
repeats no graph analysis. `limit` caps how many nodes run concurrently;
|
|
424
|
+
it defaults to `None`, which leaves concurrency unbounded (every ready
|
|
425
|
+
node runs at once). Pass an integer to cap it when the steps contend for
|
|
426
|
+
a scarce resource.
|
|
427
|
+
"""
|
|
428
|
+
if limit is not None and limit < 1:
|
|
429
|
+
raise ValueError(f"limit must be at least 1 or None, but got {limit}")
|
|
430
|
+
|
|
431
|
+
nodes = tuple(self._nodes)
|
|
432
|
+
return CompiledGraph(
|
|
433
|
+
nodes=nodes,
|
|
434
|
+
inputs=self._input_keys,
|
|
435
|
+
output=output.key,
|
|
436
|
+
limit=limit,
|
|
437
|
+
_plan=Plan.of(nodes),
|
|
438
|
+
)
|
|
File without changes
|