quantology-modelmaker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modelmaker/__init__.py +2 -0
- modelmaker/api.py +1073 -0
- modelmaker/blocks/__init__.py +0 -0
- modelmaker/blocks/base.py +78 -0
- modelmaker/blocks/library.py +327 -0
- modelmaker/blocks/modelling.py +312 -0
- modelmaker/blocks/stat_tests.py +133 -0
- modelmaker/cache.py +44 -0
- modelmaker/compiler.py +478 -0
- modelmaker/graph.py +171 -0
- modelmaker/llm/__init__.py +28 -0
- modelmaker/llm/anthropic_provider.py +79 -0
- modelmaker/llm/base.py +85 -0
- modelmaker/llm/claude_cli_provider.py +90 -0
- modelmaker/llm/gemini_provider.py +138 -0
- modelmaker/llm/lmstudio_provider.py +140 -0
- modelmaker/llm/openai_provider.py +138 -0
- modelmaker/llm/prompts.py +189 -0
- modelmaker/llm/settings.py +47 -0
- modelmaker/llm/stub_provider.py +29 -0
- modelmaker/metadata_transforms.py +109 -0
- modelmaker/packet.py +111 -0
- modelmaker/project.py +129 -0
- modelmaker/run_worker.py +143 -0
- modelmaker/runner.py +1313 -0
- modelmaker/session.py +400 -0
- modelmaker/static/assets/index-B9cHjTMW.css +1 -0
- modelmaker/static/assets/index-EvjTc-1k.js +34 -0
- modelmaker/static/brand-quantology/favicon.svg +4 -0
- modelmaker/static/brand-quantology/icon-mark-light.svg +13 -0
- modelmaker/static/icons.svg +24 -0
- modelmaker/static/index.html +20 -0
- modelmaker/tui/__init__.py +0 -0
- modelmaker/tui/app.py +785 -0
- modelmaker/tui/canvas.py +388 -0
- modelmaker/tui/charts.py +84 -0
- modelmaker/tui/client.py +270 -0
- modelmaker/tui/inspector.py +285 -0
- modelmaker/tui/paramspecs.py +128 -0
- modelmaker/tui/preview.py +71 -0
- modelmaker/tui/screens.py +182 -0
- modelmaker/util.py +44 -0
- quantology_modelmaker-0.1.0.dist-info/METADATA +247 -0
- quantology_modelmaker-0.1.0.dist-info/RECORD +49 -0
- quantology_modelmaker-0.1.0.dist-info/WHEEL +5 -0
- quantology_modelmaker-0.1.0.dist-info/entry_points.txt +3 -0
- quantology_modelmaker-0.1.0.dist-info/licenses/LICENSE +204 -0
- quantology_modelmaker-0.1.0.dist-info/licenses/NOTICE +12 -0
- quantology_modelmaker-0.1.0.dist-info/top_level.txt +1 -0
modelmaker/compiler.py
ADDED
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import textwrap
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
|
|
7
|
+
from .blocks.base import BLOCK_REGISTRY, BlockSpec
|
|
8
|
+
from .graph import Graph
|
|
9
|
+
from .packet import ColumnRole, DataFramePacket, resolve_role_column
|
|
10
|
+
from .runner import FusionGroup, _is_fusable_block
|
|
11
|
+
from .util import ROLE_PARAM_NAMES, accepts_param, find_role_param
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CompileError(Exception):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _sanitize(name: str) -> str:
|
|
19
|
+
out = "".join(c if c.isalnum() or c == "_" else "_" for c in name)
|
|
20
|
+
return out if out and not out[0].isdigit() else f"_{out}"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _alloc_output_vars(
|
|
24
|
+
bid: str,
|
|
25
|
+
block,
|
|
26
|
+
out_ports: list[str],
|
|
27
|
+
var_names: dict[tuple[str, str], str],
|
|
28
|
+
used_var_names: set[str],
|
|
29
|
+
) -> list[str]:
|
|
30
|
+
"""Pick the compiled script's variable name(s) for `bid`'s output
|
|
31
|
+
port(s) -- a user-chosen port name (see BlockInstance.port_names) when
|
|
32
|
+
one's set and not already taken, else the block-name/id/port scheme,
|
|
33
|
+
which is always unique by construction. Shared between the ordinary
|
|
34
|
+
per-block call emission and a fused group's exit(s) (see
|
|
35
|
+
_fused_group_call_lines) so a name picked for a streaming exit reads
|
|
36
|
+
exactly like any other block's output would."""
|
|
37
|
+
varlist = []
|
|
38
|
+
for p in out_ports:
|
|
39
|
+
custom = block.port_names.get(p)
|
|
40
|
+
var = _sanitize(custom) if custom else None
|
|
41
|
+
if var is None or var in used_var_names:
|
|
42
|
+
var = f"{_sanitize(block.name)}_{bid}" if len(out_ports) == 1 else f"{_sanitize(block.name)}_{bid}_{p}"
|
|
43
|
+
used_var_names.add(var)
|
|
44
|
+
var_names[(bid, p)] = var
|
|
45
|
+
varlist.append(var)
|
|
46
|
+
return varlist
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _build_compile_fusion_groups(graph: Graph, order: list[str], must_exit: set[str]) -> list[FusionGroup]:
|
|
50
|
+
"""Compiler-side counterpart of Runner._build_fusion_groups (see
|
|
51
|
+
runner.py for the full design): groups `order` (the whole reachable
|
|
52
|
+
compile set -- a compiled script unconditionally (re)executes
|
|
53
|
+
everything it emits, so there's no "not yet current" subset the way a
|
|
54
|
+
live run has) into fusion groups exactly as FusionGroup describes,
|
|
55
|
+
using dataframe wires restricted to `order` itself -- a block outside
|
|
56
|
+
what this compile emits doesn't exist as far as the script is
|
|
57
|
+
concerned, unlike the live engine where an already-cached consumer
|
|
58
|
+
outside the run still matters for a future run.
|
|
59
|
+
|
|
60
|
+
`must_exit` (an explicitly requested `output_blocks` entry) forces a
|
|
61
|
+
real, individually-named exit even when the block's only consumer is
|
|
62
|
+
also being compiled -- the whole point of naming a block in
|
|
63
|
+
`output_blocks` is to get its own variable in the script."""
|
|
64
|
+
candidate_set = set(order)
|
|
65
|
+
fusable: dict[str, BlockSpec] = {}
|
|
66
|
+
for bid in order:
|
|
67
|
+
spec = _is_fusable_block(graph.blocks[bid])
|
|
68
|
+
if spec is not None:
|
|
69
|
+
fusable[bid] = spec
|
|
70
|
+
|
|
71
|
+
parent: dict[str, str] = {bid: bid for bid in fusable}
|
|
72
|
+
|
|
73
|
+
def find(x: str) -> str:
|
|
74
|
+
while parent[x] != x:
|
|
75
|
+
parent[x] = parent[parent[x]]
|
|
76
|
+
x = parent[x]
|
|
77
|
+
return x
|
|
78
|
+
|
|
79
|
+
def union(a: str, b: str) -> None:
|
|
80
|
+
ra, rb = find(a), find(b)
|
|
81
|
+
if ra != rb:
|
|
82
|
+
parent[ra] = rb
|
|
83
|
+
|
|
84
|
+
port_types_cache: dict[str, dict[str, str]] = {}
|
|
85
|
+
|
|
86
|
+
def port_types(bid: str) -> dict[str, str]:
|
|
87
|
+
if bid not in port_types_cache:
|
|
88
|
+
port_types_cache[bid] = {p.name: p.type for p in graph.blocks[bid].inputs}
|
|
89
|
+
return port_types_cache[bid]
|
|
90
|
+
|
|
91
|
+
for bid in fusable:
|
|
92
|
+
for port, wire in graph.input_wires(bid).items():
|
|
93
|
+
if port_types(bid).get(port) != "dataframe":
|
|
94
|
+
continue
|
|
95
|
+
if wire.from_block in fusable:
|
|
96
|
+
union(bid, wire.from_block)
|
|
97
|
+
|
|
98
|
+
groups_by_root: dict[str, list[str]] = {}
|
|
99
|
+
for bid in order: # preserves topo order
|
|
100
|
+
key = find(bid) if bid in fusable else bid
|
|
101
|
+
groups_by_root.setdefault(key, []).append(bid)
|
|
102
|
+
|
|
103
|
+
groups: list[FusionGroup] = []
|
|
104
|
+
for members in groups_by_root.values():
|
|
105
|
+
member_set = set(members)
|
|
106
|
+
if len(members) == 1 and members[0] not in fusable:
|
|
107
|
+
groups.append(FusionGroup(members=list(members), exits=set(member_set)))
|
|
108
|
+
continue
|
|
109
|
+
exits: set[str] = set()
|
|
110
|
+
for bid in members:
|
|
111
|
+
out_ports = {p.name for p in graph.blocks[bid].outputs if p.type == "dataframe"}
|
|
112
|
+
df_consumers = [
|
|
113
|
+
w
|
|
114
|
+
for w in graph.wires.values()
|
|
115
|
+
if w.from_block == bid
|
|
116
|
+
and w.from_port in out_ports
|
|
117
|
+
and w.to_block in candidate_set
|
|
118
|
+
and port_types(w.to_block).get(w.to_port) == "dataframe"
|
|
119
|
+
]
|
|
120
|
+
if not df_consumers or any(w.to_block not in member_set for w in df_consumers) or bid in must_exit:
|
|
121
|
+
exits.add(bid)
|
|
122
|
+
groups.append(FusionGroup(members=list(members), exits=exits))
|
|
123
|
+
return groups
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _fused_group_call_lines(
|
|
127
|
+
group: FusionGroup,
|
|
128
|
+
graph: Graph,
|
|
129
|
+
fn_names: dict[str, str],
|
|
130
|
+
var_names: dict[tuple[str, str], str],
|
|
131
|
+
used_var_names: set[str],
|
|
132
|
+
) -> list[str]:
|
|
133
|
+
"""Emit one fusion group (see _build_compile_fusion_groups) as a
|
|
134
|
+
single polars lazy plan: every member becomes a `_lz_<id> = fn(...)`
|
|
135
|
+
assignment (chained inputs referencing another member's own `_lz_`
|
|
136
|
+
variable directly; anything else -- an already-compiled eager
|
|
137
|
+
DataFrame from outside the group -- wrapped in `.lazy()` so it joins
|
|
138
|
+
the same unbroken plan), and the group's exit(s) are collected
|
|
139
|
+
together in one `pl.collect_all(..., engine="streaming")` call, which
|
|
140
|
+
applies common-subplan elimination across them so a shared upstream
|
|
141
|
+
stretch (fan-out) is computed once even when it feeds two exits."""
|
|
142
|
+
member_set = set(group.members)
|
|
143
|
+
lines: list[str] = [f"# --- Fused (streaming): {', '.join(group.members)} ---"]
|
|
144
|
+
lazy_var = {bid: f"_lz_{_sanitize(bid)}" for bid in group.members}
|
|
145
|
+
|
|
146
|
+
for bid in group.members:
|
|
147
|
+
block = graph.blocks[bid]
|
|
148
|
+
kwarg_pairs: list[tuple[str, str]] = []
|
|
149
|
+
for port, wire in sorted(graph.input_wires(bid).items()):
|
|
150
|
+
if wire.from_block in member_set:
|
|
151
|
+
kwarg_pairs.append((port, lazy_var[wire.from_block]))
|
|
152
|
+
else:
|
|
153
|
+
kwarg_pairs.append((port, f"{var_names[(wire.from_block, wire.from_port)]}.lazy()"))
|
|
154
|
+
for pname, pval in block.params.items():
|
|
155
|
+
kwarg_pairs.append((pname, repr(pval)))
|
|
156
|
+
call = f"{fn_names[bid]}({', '.join(f'{k}={v}' for k, v in kwarg_pairs)})"
|
|
157
|
+
lines.append(f"{lazy_var[bid]} = {call}")
|
|
158
|
+
|
|
159
|
+
exits = [bid for bid in group.members if bid in group.exits]
|
|
160
|
+
exit_vars = [_alloc_output_vars(bid, graph.blocks[bid], ["out"], var_names, used_var_names)[0] for bid in exits]
|
|
161
|
+
collect_call = f"pl.collect_all([{', '.join(lazy_var[bid] for bid in exits)}], engine='streaming')"
|
|
162
|
+
if len(exit_vars) == 1:
|
|
163
|
+
lines.append(f"({exit_vars[0]},) = {collect_call}")
|
|
164
|
+
else:
|
|
165
|
+
lines.append(f"{', '.join(exit_vars)} = {collect_call}")
|
|
166
|
+
return lines
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _resolve_role_value(graph: Graph, runner, block_id: str, role: ColumnRole) -> str | None:
|
|
170
|
+
"""Same dynamic default as Runner.run_block (see packet.resolve_role_column),
|
|
171
|
+
computed here from the already-cached, already-green upstream packets
|
|
172
|
+
`runner` holds (compiling requires every reachable block to be green
|
|
173
|
+
first -- see the not_ready check above) rather than re-running anything.
|
|
174
|
+
Retagging a column's role changes the block's cache key (see
|
|
175
|
+
Runner.compute_key), so a block whose target/predicted column just moved
|
|
176
|
+
is grey again and (in strict mode) blocks compilation until it's re-run
|
|
177
|
+
-- by the time this runs, the cached packet always reflects the current
|
|
178
|
+
tag."""
|
|
179
|
+
if runner is None:
|
|
180
|
+
return None
|
|
181
|
+
schema_metas = []
|
|
182
|
+
for wire in graph.input_wires(block_id).values():
|
|
183
|
+
st = runner.state.get(wire.from_block)
|
|
184
|
+
if st is None or st.last_successful_key is None:
|
|
185
|
+
continue
|
|
186
|
+
entry = runner.cache.get(st.last_successful_key)
|
|
187
|
+
if entry is None:
|
|
188
|
+
continue
|
|
189
|
+
packet = entry.outputs.get(wire.from_port)
|
|
190
|
+
if isinstance(packet, DataFramePacket):
|
|
191
|
+
schema_metas.append(packet.schema_meta)
|
|
192
|
+
return resolve_role_column(schema_metas, role)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def compile_graph(
|
|
196
|
+
graph: Graph,
|
|
197
|
+
runner=None,
|
|
198
|
+
output_blocks: list[str] | None = None,
|
|
199
|
+
strict: bool = True,
|
|
200
|
+
stream: bool = False,
|
|
201
|
+
) -> str:
|
|
202
|
+
"""Compile the graph to a single, self-contained Python script (plan
|
|
203
|
+
section 7, default/clean mode — DataFramePacket/ColumnMeta scaffolding
|
|
204
|
+
is absent by construction, since block bodies are plain-dataframe
|
|
205
|
+
functions to begin with). `--with-metadata` verbose mode is not yet
|
|
206
|
+
implemented.
|
|
207
|
+
|
|
208
|
+
`stream=True` mirrors Runner.run_all_streaming for a compiled script:
|
|
209
|
+
whatever connected stretch of fusable blocks (filter/select/
|
|
210
|
+
groupby_agg/join/read_csv -- see BlockSpec.lazy_fn, and
|
|
211
|
+
_build_compile_fusion_groups for the exact same grouping rule the
|
|
212
|
+
live engine uses) is emitted as one polars lazy plan, collected once
|
|
213
|
+
via `pl.collect_all(..., engine="streaming")`, instead of one eager
|
|
214
|
+
call per block -- so a compiled script's source, not just the live
|
|
215
|
+
engine, can process a source larger than memory. Default False keeps
|
|
216
|
+
every existing compiled script byte-for-byte identical to before this
|
|
217
|
+
existed."""
|
|
218
|
+
targets = output_blocks or list(graph.blocks)
|
|
219
|
+
reachable = graph.ancestors(targets)
|
|
220
|
+
|
|
221
|
+
if runner is not None:
|
|
222
|
+
not_ready = sorted(b for b in reachable if runner.status(b) in ("grey", "red"))
|
|
223
|
+
if not_ready and strict:
|
|
224
|
+
raise CompileError(f"blocks not ready to compile (grey/red): {not_ready}")
|
|
225
|
+
|
|
226
|
+
order = [b for b in graph.topo_order() if b in reachable]
|
|
227
|
+
|
|
228
|
+
# Streaming mode groups the compile set up front so both passes below
|
|
229
|
+
# (function defs, then call sites) can tell a fused member from an
|
|
230
|
+
# ordinary one -- must_exit only applies when output_blocks was given
|
|
231
|
+
# explicitly (an unscoped compile has no extra "name this variable"
|
|
232
|
+
# requirement beyond normal graph consumption, which the grouping's
|
|
233
|
+
# own no-consumers-outside-the-group rule already covers).
|
|
234
|
+
fusion_groups = _build_compile_fusion_groups(graph, order, set(output_blocks or [])) if stream else []
|
|
235
|
+
fused_group_of: dict[str, FusionGroup] = {}
|
|
236
|
+
for group in fusion_groups:
|
|
237
|
+
if len(group.members) > 1:
|
|
238
|
+
for bid in group.members:
|
|
239
|
+
fused_group_of[bid] = group
|
|
240
|
+
|
|
241
|
+
fn_names: dict[str, str] = {}
|
|
242
|
+
wants_output_dir: dict[str, bool] = {}
|
|
243
|
+
wants_block_id: dict[str, bool] = {}
|
|
244
|
+
wants_role_params: dict[str, dict[ColumnRole, str]] = {}
|
|
245
|
+
def_lines: list[str] = []
|
|
246
|
+
# Two blocks that run the same code (same category, for registry blocks;
|
|
247
|
+
# same category *and* code text, for custom ones) get exactly one
|
|
248
|
+
# function definition, shared across every instance's call site below --
|
|
249
|
+
# e.g. two WoE blocks compile to one `woe_transform` function called
|
|
250
|
+
# twice with each instance's own params, not two near-identical copies.
|
|
251
|
+
# A fused member's def uses its lazy_fn instead of the eager fn (see
|
|
252
|
+
# BlockSpec.lazy_fn) -- for filter/select/groupby_agg/join that's
|
|
253
|
+
# literally the same function object, so the emitted source is
|
|
254
|
+
# unchanged; only read_csv's separate lazy twin actually differs, and
|
|
255
|
+
# naturally gets its own def (different source text -> different key
|
|
256
|
+
# below) alongside any non-fused read_csv elsewhere in the script.
|
|
257
|
+
seen_fns: dict[tuple[str, str], str] = {}
|
|
258
|
+
used_fn_names: set[str] = set()
|
|
259
|
+
for bid in order:
|
|
260
|
+
block = graph.blocks[bid]
|
|
261
|
+
group = fused_group_of.get(bid)
|
|
262
|
+
if group is not None:
|
|
263
|
+
fn = BLOCK_REGISTRY[block.category].lazy_fn
|
|
264
|
+
else:
|
|
265
|
+
fn = block.resolved_fn()
|
|
266
|
+
wants_output_dir[bid] = accepts_param(fn, "output_dir")
|
|
267
|
+
wants_block_id[bid] = accepts_param(fn, "block_id")
|
|
268
|
+
wants_role_params[bid] = {
|
|
269
|
+
role: role_param for role in ROLE_PARAM_NAMES if (role_param := find_role_param(fn, role)) is not None
|
|
270
|
+
}
|
|
271
|
+
src = block.code if block.is_custom else inspect.getsource(fn)
|
|
272
|
+
src = textwrap.dedent(src).strip("\n")
|
|
273
|
+
fn_sig = (block.category, src)
|
|
274
|
+
existing_fn_name = seen_fns.get(fn_sig)
|
|
275
|
+
if existing_fn_name is not None:
|
|
276
|
+
fn_names[bid] = existing_fn_name
|
|
277
|
+
continue
|
|
278
|
+
|
|
279
|
+
# A custom (AI-authored) block's function is named after the block
|
|
280
|
+
# itself -- the same human-chosen name already used for its call-site
|
|
281
|
+
# variable below -- rather than its auto-generated category (e.g.
|
|
282
|
+
# "ai_block_lz3k9f"), so the compiled def reads like the block does
|
|
283
|
+
# in the UI. Registry blocks keep the plain category name (a stable,
|
|
284
|
+
# well-known function name shared by every instance); a different
|
|
285
|
+
# *bodied* block already claiming it only bites two custom blocks
|
|
286
|
+
# that happen to share a category but were drafted with different
|
|
287
|
+
# bodies, where the block id disambiguates them.
|
|
288
|
+
base_name = _sanitize(block.name) if block.is_custom else _sanitize(block.category)
|
|
289
|
+
fn_name = base_name if base_name not in used_fn_names else f"{base_name}_{bid}"
|
|
290
|
+
used_fn_names.add(fn_name)
|
|
291
|
+
seen_fns[fn_sig] = fn_name
|
|
292
|
+
fn_names[bid] = fn_name
|
|
293
|
+
src = src.replace(f"def {fn.__name__}(", f"def {fn_name}(", 1)
|
|
294
|
+
def_lines.append(f"# === Function: {fn_name} | type={block.block_type} | category={block.category} ===")
|
|
295
|
+
def_lines.append(src)
|
|
296
|
+
def_lines.append("")
|
|
297
|
+
def_lines.append("")
|
|
298
|
+
|
|
299
|
+
call_lines: list[str] = []
|
|
300
|
+
var_names: dict[tuple[str, str], str] = {}
|
|
301
|
+
used_var_names: set[str] = set()
|
|
302
|
+
current_lane: str | None = "__unset__"
|
|
303
|
+
uses_group_by = any(graph.blocks[bid].group_by for bid in order)
|
|
304
|
+
emitted_groups: set[int] = set()
|
|
305
|
+
for bid in order:
|
|
306
|
+
block = graph.blocks[bid]
|
|
307
|
+
# Lanes group the pipeline into modeling phases (Data Prep -> Feature
|
|
308
|
+
# Engineering -> ...); a banner marks each phase's calls in the
|
|
309
|
+
# compiled script, purely for readability -- lane membership plays no
|
|
310
|
+
# role in execution order (see graph.topo_order).
|
|
311
|
+
lane_name = graph.lanes[block.lane].name if block.lane in graph.lanes else None
|
|
312
|
+
if lane_name != current_lane:
|
|
313
|
+
current_lane = lane_name
|
|
314
|
+
if lane_name:
|
|
315
|
+
call_lines.append(f"# ===== Lane: {lane_name} =====")
|
|
316
|
+
|
|
317
|
+
group = fused_group_of.get(bid)
|
|
318
|
+
if group is not None:
|
|
319
|
+
if id(group) in emitted_groups:
|
|
320
|
+
continue # already emitted in full when we reached the group's first member
|
|
321
|
+
emitted_groups.add(id(group))
|
|
322
|
+
call_lines.extend(_fused_group_call_lines(group, graph, fn_names, var_names, used_var_names))
|
|
323
|
+
call_lines.append("")
|
|
324
|
+
continue
|
|
325
|
+
|
|
326
|
+
# (kwarg name, source-code expression for its value) pairs -- kept
|
|
327
|
+
# apart instead of pre-joined into "name=value" text so grouped
|
|
328
|
+
# blocks (below) can split dataframe-typed wired inputs (candidates
|
|
329
|
+
# for partitioning) from everything else (params, role defaults,
|
|
330
|
+
# output_dir/block_id -- unchanged across groups).
|
|
331
|
+
dataframe_ports = {p.name for p in block.inputs if p.type == "dataframe"}
|
|
332
|
+
kwarg_pairs: list[tuple[str, str]] = []
|
|
333
|
+
for port, wire in sorted(graph.input_wires(bid).items()):
|
|
334
|
+
kwarg_pairs.append((port, var_names[(wire.from_block, wire.from_port)]))
|
|
335
|
+
for pname, pval in block.params.items():
|
|
336
|
+
kwarg_pairs.append((pname, repr(pval)))
|
|
337
|
+
for role, role_param in wants_role_params[bid].items():
|
|
338
|
+
if role_param not in block.params:
|
|
339
|
+
resolved = _resolve_role_value(graph, runner, bid, role)
|
|
340
|
+
if resolved is not None:
|
|
341
|
+
kwarg_pairs.append((role_param, repr(resolved)))
|
|
342
|
+
if block.block_type == "output" and wants_output_dir[bid]:
|
|
343
|
+
kwarg_pairs.append(("output_dir", "OUTPUT_DIR"))
|
|
344
|
+
if block.block_type == "output" and wants_block_id[bid]:
|
|
345
|
+
kwarg_pairs.append(("block_id", repr(bid)))
|
|
346
|
+
|
|
347
|
+
out_ports = [p.name for p in block.outputs]
|
|
348
|
+
call_lines.append(f'# --- Call: {bid} | name="{block.name}" ---')
|
|
349
|
+
|
|
350
|
+
# A port the user's named (see BlockInstance.port_names, settable
|
|
351
|
+
# by clicking that output's data in the UI) becomes the variable
|
|
352
|
+
# holding it here, so the compiled script reads with the same
|
|
353
|
+
# names the user gave the data -- falling back to the
|
|
354
|
+
# block-name/id/port scheme, which is always unique by
|
|
355
|
+
# construction, for any port left unnamed or whose chosen name
|
|
356
|
+
# collides with another one already used in this script.
|
|
357
|
+
varlist = _alloc_output_vars(bid, block, out_ports, var_names, used_var_names)
|
|
358
|
+
|
|
359
|
+
if block.group_by:
|
|
360
|
+
call_lines.extend(_grouped_call_lines(bid, block, fn_names[bid], dataframe_ports, kwarg_pairs, varlist))
|
|
361
|
+
else:
|
|
362
|
+
call = f"{fn_names[bid]}({', '.join(f'{k}={v}' for k, v in kwarg_pairs)})"
|
|
363
|
+
if not out_ports:
|
|
364
|
+
call_lines.append(call)
|
|
365
|
+
else:
|
|
366
|
+
call_lines.append(f"{', '.join(varlist)} = {call}" if len(varlist) > 1 else f"{varlist[0]} = {call}")
|
|
367
|
+
call_lines.append("")
|
|
368
|
+
|
|
369
|
+
header = [
|
|
370
|
+
"# Generated by Model-Maker. Do not edit function bodies here directly --",
|
|
371
|
+
"# edit the source blocks in the project and recompile.",
|
|
372
|
+
f"# Compiled: {datetime.now(timezone.utc).isoformat()}",
|
|
373
|
+
"",
|
|
374
|
+
"import os",
|
|
375
|
+
"import polars as pl",
|
|
376
|
+
]
|
|
377
|
+
if uses_group_by:
|
|
378
|
+
header.append("from concurrent.futures import ThreadPoolExecutor")
|
|
379
|
+
header += [
|
|
380
|
+
"",
|
|
381
|
+
'OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "./output")',
|
|
382
|
+
"os.makedirs(OUTPUT_DIR, exist_ok=True)",
|
|
383
|
+
"",
|
|
384
|
+
"",
|
|
385
|
+
]
|
|
386
|
+
if uses_group_by:
|
|
387
|
+
header.append(_COMBINE_GROUP_RESULTS_SRC)
|
|
388
|
+
header.append("")
|
|
389
|
+
|
|
390
|
+
return "\n".join(header + def_lines + call_lines) + "\n"
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
# A grouped block's per-group calls each return whatever the block normally
|
|
394
|
+
# returns (a single value, or a tuple for a multi-output block) -- this
|
|
395
|
+
# reshapes {group_value: that return value} into the single/tuple result
|
|
396
|
+
# the rest of the script expects, exactly mirroring Runner._combine_group_results
|
|
397
|
+
# so a compiled script's grouped output matches the live engine's. Emitted
|
|
398
|
+
# once, only when at least one block in the graph uses group_by.
|
|
399
|
+
_COMBINE_GROUP_RESULTS_SRC = '''def _combine_group_results(group_col, group_values, raw_by_group, out_names):
|
|
400
|
+
if not out_names:
|
|
401
|
+
return None
|
|
402
|
+
per_group = {}
|
|
403
|
+
for gval in group_values:
|
|
404
|
+
raw = raw_by_group[gval]
|
|
405
|
+
per_group[gval] = (raw,) if len(out_names) == 1 else tuple(raw)
|
|
406
|
+
combined = []
|
|
407
|
+
for i in range(len(out_names)):
|
|
408
|
+
values = [per_group[g][i] for g in group_values]
|
|
409
|
+
if all(isinstance(v, pl.DataFrame) for v in values):
|
|
410
|
+
parts = [
|
|
411
|
+
v if group_col in v.columns else v.with_columns(pl.lit(gval).alias(group_col))
|
|
412
|
+
for gval, v in zip(group_values, values)
|
|
413
|
+
]
|
|
414
|
+
combined.append(pl.concat(parts, how="diagonal_relaxed"))
|
|
415
|
+
elif all(isinstance(v, dict) for v in values):
|
|
416
|
+
combined.append(pl.DataFrame([{group_col: gval, **v} for gval, v in zip(group_values, values)]))
|
|
417
|
+
else:
|
|
418
|
+
combined.append(dict(zip(group_values, values)))
|
|
419
|
+
return combined[0] if len(combined) == 1 else tuple(combined)
|
|
420
|
+
'''
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _grouped_call_lines(
|
|
424
|
+
bid: str,
|
|
425
|
+
block,
|
|
426
|
+
fn_name: str,
|
|
427
|
+
dataframe_ports: set[str],
|
|
428
|
+
kwarg_pairs: list[tuple[str, str]],
|
|
429
|
+
varlist: list[str],
|
|
430
|
+
) -> list[str]:
|
|
431
|
+
"""Emit a group_by block's call as a partition/dispatch/recombine
|
|
432
|
+
sequence instead of a single call -- mirrors Runner._run_grouped, but
|
|
433
|
+
threaded (ThreadPoolExecutor) rather than process-isolated: this is a
|
|
434
|
+
one-shot batch script the user runs themselves, not a long-lived server
|
|
435
|
+
process other work needs protecting from, so the extra isolation the
|
|
436
|
+
live app pays for isn't worth the portability cost here (a compiled
|
|
437
|
+
script using real subprocesses would need every grouped block wrapped
|
|
438
|
+
in `if __name__ == "__main__":` to be safe under Windows' default
|
|
439
|
+
"spawn" start method)."""
|
|
440
|
+
df_pairs = [(k, v) for k, v in kwarg_pairs if k in dataframe_ports]
|
|
441
|
+
other_pairs = [(k, v) for k, v in kwarg_pairs if k not in dataframe_ports]
|
|
442
|
+
group_col = block.group_by
|
|
443
|
+
max_workers = block.max_workers or 4
|
|
444
|
+
suffix = bid
|
|
445
|
+
|
|
446
|
+
lines = [
|
|
447
|
+
f"_df_kwargs_{suffix} = {{{', '.join(f'{k!r}: {v}' for k, v in df_pairs)}}}",
|
|
448
|
+
f"_fixed_kwargs_{suffix} = {{{', '.join(f'{k!r}: {v}' for k, v in other_pairs)}}}",
|
|
449
|
+
f"_groupable_{suffix} = {{k: v for k, v in _df_kwargs_{suffix}.items() if {group_col!r} in v.columns}}",
|
|
450
|
+
f"if not _groupable_{suffix}:",
|
|
451
|
+
f" raise RuntimeError({f'group_by column {group_col!r} not found in any dataframe input of block {bid!r}'!r})",
|
|
452
|
+
f"_primary_port_{suffix} = next(iter(_groupable_{suffix}))",
|
|
453
|
+
f"_n_groups_{suffix} = _groupable_{suffix}[_primary_port_{suffix}].select(pl.col({group_col!r}).n_unique()).item() or 0",
|
|
454
|
+
f"if _n_groups_{suffix} == 0:",
|
|
455
|
+
f" raise RuntimeError({f'no groups found for group_by column {group_col!r}'!r})",
|
|
456
|
+
f"if _n_groups_{suffix} > 500:",
|
|
457
|
+
f" raise RuntimeError(f\"grouping by {group_col!r} would produce {{_n_groups_{suffix}}} groups, over the 500-group safety limit\")",
|
|
458
|
+
f"_partitions_{suffix} = {{"
|
|
459
|
+
f"port: {{k[0]: v for k, v in frame.partition_by({group_col!r}, as_dict=True).items()}} "
|
|
460
|
+
f"for port, frame in _groupable_{suffix}.items()}}",
|
|
461
|
+
f"_group_values_{suffix} = list(_partitions_{suffix}[_primary_port_{suffix}].keys())",
|
|
462
|
+
"",
|
|
463
|
+
f"def _task_{suffix}(_gval):",
|
|
464
|
+
f" _kwargs = dict(_df_kwargs_{suffix})",
|
|
465
|
+
f" _kwargs.update(_fixed_kwargs_{suffix})",
|
|
466
|
+
f" for _port, _parts in _partitions_{suffix}.items():",
|
|
467
|
+
" if _gval in _parts:",
|
|
468
|
+
" _kwargs[_port] = _parts[_gval]",
|
|
469
|
+
f" return {fn_name}(**_kwargs)",
|
|
470
|
+
"",
|
|
471
|
+
f"with ThreadPoolExecutor(max_workers=min({max_workers}, _n_groups_{suffix})) as _ex:",
|
|
472
|
+
f" _raw_{suffix} = dict(zip(_group_values_{suffix}, _ex.map(_task_{suffix}, _group_values_{suffix})))",
|
|
473
|
+
"",
|
|
474
|
+
f"_combined_{suffix} = _combine_group_results({group_col!r}, _group_values_{suffix}, _raw_{suffix}, {varlist!r})",
|
|
475
|
+
]
|
|
476
|
+
if varlist:
|
|
477
|
+
lines.append(f"{', '.join(varlist)} = _combined_{suffix}" if len(varlist) > 1 else f"{varlist[0]} = _combined_{suffix}")
|
|
478
|
+
return lines
|
modelmaker/graph.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import types
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import polars as pl
|
|
8
|
+
|
|
9
|
+
from .blocks.base import BLOCK_REGISTRY, BlockType, PortSpec
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Position:
|
|
14
|
+
x: float = 0
|
|
15
|
+
y: float = 0
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Lane:
|
|
20
|
+
name: str
|
|
21
|
+
order: int
|
|
22
|
+
# Pixel height of this lane's band on the canvas -- user-resizable by
|
|
23
|
+
# dragging its top/bottom border (see LaneBand.tsx). Lanes stack
|
|
24
|
+
# end-to-end by `order` with no gap, so lanes can never overlap by
|
|
25
|
+
# construction; resizing one only shifts the ones after it.
|
|
26
|
+
height: float = 260.0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Wire:
|
|
31
|
+
id: str
|
|
32
|
+
from_block: str
|
|
33
|
+
from_port: str
|
|
34
|
+
to_block: str
|
|
35
|
+
to_port: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class BlockInstance:
|
|
40
|
+
id: str
|
|
41
|
+
block_type: BlockType
|
|
42
|
+
category: str
|
|
43
|
+
name: str
|
|
44
|
+
lane: str | None
|
|
45
|
+
position: Position
|
|
46
|
+
code_version: int
|
|
47
|
+
params: dict[str, Any]
|
|
48
|
+
inputs: list[PortSpec]
|
|
49
|
+
outputs: list[PortSpec]
|
|
50
|
+
code_ref: str | None = None
|
|
51
|
+
code: str | None = None
|
|
52
|
+
# dict declaration for custom blocks (see metadata_transforms.resolve);
|
|
53
|
+
# None for registry blocks, whose transform lives in the registry.
|
|
54
|
+
metadata_transform: dict[str, Any] | None = None
|
|
55
|
+
# True for an AI-authored block with its own editable code/metadata_transform;
|
|
56
|
+
# False for a block backed by a fixed BLOCK_REGISTRY entry. Independent of
|
|
57
|
+
# block_type -- a custom block can be an input, standard, or output block.
|
|
58
|
+
is_custom: bool = False
|
|
59
|
+
# column name -> ColumnRole value (see packet.py), hand-tagged by the
|
|
60
|
+
# user (see session.set_column_role) and layered on top of whatever role
|
|
61
|
+
# this block's own metadata_transform already computed -- applies to
|
|
62
|
+
# that column on every output port it appears on. Included in
|
|
63
|
+
# Runner.compute_key's hash basis alongside params/code, so retagging a
|
|
64
|
+
# column invalidates this block and everything downstream the same way
|
|
65
|
+
# any other param change does; no separate staleness bookkeeping needed.
|
|
66
|
+
column_role_overrides: dict[str, str] = field(default_factory=dict)
|
|
67
|
+
# output port name -> user-given name for the data on that port (e.g.
|
|
68
|
+
# "raw applications"), settable by clicking that port's data in the UI
|
|
69
|
+
# (see PortInspector.tsx). Purely descriptive except in one place: the
|
|
70
|
+
# compiler uses it, when set, as the compiled variable name that holds
|
|
71
|
+
# this port's data instead of the auto-generated block-name/id scheme
|
|
72
|
+
# (see compiler.compile_graph) -- so naming your data also names it in
|
|
73
|
+
# the generated script.
|
|
74
|
+
port_names: dict[str, str] = field(default_factory=dict)
|
|
75
|
+
# When set, this block runs once per distinct value of this column
|
|
76
|
+
# (present in one of its dataframe inputs) instead of once overall --
|
|
77
|
+
# e.g. an auc_gini block with group_by="region" produces a Gini per
|
|
78
|
+
# region instead of one for the whole dataset. See Runner._run_grouped.
|
|
79
|
+
# None (the default) is plain, ungrouped execution.
|
|
80
|
+
group_by: str | None = None
|
|
81
|
+
# Cap on how many of this block's groups may run concurrently, when
|
|
82
|
+
# group_by is set. None means "use the runner's default" (see
|
|
83
|
+
# Runner.MAX_GROUP_WORKERS). Ignored for ungrouped blocks.
|
|
84
|
+
max_workers: int | None = None
|
|
85
|
+
|
|
86
|
+
def resolved_fn(self):
|
|
87
|
+
if self.is_custom:
|
|
88
|
+
return _compile_code_to_fn(self.code or "")
|
|
89
|
+
return BLOCK_REGISTRY[self.category].fn
|
|
90
|
+
|
|
91
|
+
def resolved_metadata_transform(self):
|
|
92
|
+
if self.is_custom:
|
|
93
|
+
return self.metadata_transform
|
|
94
|
+
return BLOCK_REGISTRY[self.category].metadata_transform
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _compile_code_to_fn(code: str):
|
|
98
|
+
# `pl` is available here to match the compiled script's module-level
|
|
99
|
+
# `import polars as pl` (see compiler.py) -- custom block code should be
|
|
100
|
+
# able to rely on it being in scope either way.
|
|
101
|
+
ns: dict[str, Any] = {"pl": pl}
|
|
102
|
+
exec(code, ns) # trusted, project-local code; matches the plan's no-sandboxing execution model
|
|
103
|
+
fns = [v for v in ns.values() if isinstance(v, types.FunctionType)]
|
|
104
|
+
if len(fns) != 1:
|
|
105
|
+
raise ValueError("custom block code must define exactly one top-level function")
|
|
106
|
+
return fns[0]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass
|
|
110
|
+
class Graph:
|
|
111
|
+
lanes: dict[str, Lane] = field(default_factory=dict)
|
|
112
|
+
blocks: dict[str, BlockInstance] = field(default_factory=dict)
|
|
113
|
+
wires: dict[str, Wire] = field(default_factory=dict)
|
|
114
|
+
|
|
115
|
+
def input_wires(self, block_id: str) -> dict[str, Wire]:
|
|
116
|
+
return {w.to_port: w for w in self.wires.values() if w.to_block == block_id}
|
|
117
|
+
|
|
118
|
+
def predecessors(self, block_id: str) -> set[str]:
|
|
119
|
+
return {w.from_block for w in self.wires.values() if w.to_block == block_id}
|
|
120
|
+
|
|
121
|
+
def successors(self, block_id: str) -> set[str]:
|
|
122
|
+
return {w.to_block for w in self.wires.values() if w.from_block == block_id}
|
|
123
|
+
|
|
124
|
+
def ancestors(self, block_ids: list[str]) -> set[str]:
|
|
125
|
+
seen: set[str] = set()
|
|
126
|
+
stack = list(block_ids)
|
|
127
|
+
while stack:
|
|
128
|
+
b = stack.pop()
|
|
129
|
+
if b in seen:
|
|
130
|
+
continue
|
|
131
|
+
seen.add(b)
|
|
132
|
+
stack.extend(self.predecessors(b))
|
|
133
|
+
return seen
|
|
134
|
+
|
|
135
|
+
def creates_cycle(self, from_block: str, to_block: str) -> bool:
|
|
136
|
+
"""Would a wire from_block -> to_block create a cycle? True if
|
|
137
|
+
to_block is already an ancestor of from_block (including
|
|
138
|
+
from_block == to_block), i.e. a path to_block -> ... -> from_block
|
|
139
|
+
already exists, so adding from_block -> to_block would close a loop.
|
|
140
|
+
Checked before a wire is added -- see session.add_wire -- so the
|
|
141
|
+
graph a user can build is acyclic by construction and topo_order's
|
|
142
|
+
cycle branch is unreachable in normal use."""
|
|
143
|
+
return to_block in self.ancestors([from_block])
|
|
144
|
+
|
|
145
|
+
def topo_order(self) -> list[str]:
|
|
146
|
+
"""Deterministic topological sort: ties broken by (lane order, canvas
|
|
147
|
+
position, block id) per plan section 7 — lanes play no role in
|
|
148
|
+
execution order itself, only in tie-breaking for stable output.
|
|
149
|
+
Blocks are ordered by y before x since flow runs top-to-bottom, to
|
|
150
|
+
match visual reading order for otherwise-unordered blocks."""
|
|
151
|
+
visited: set[str] = set()
|
|
152
|
+
order: list[str] = []
|
|
153
|
+
|
|
154
|
+
def sort_key(bid: str):
|
|
155
|
+
b = self.blocks[bid]
|
|
156
|
+
lane_order = self.lanes[b.lane].order if b.lane in self.lanes else 0
|
|
157
|
+
return (lane_order, b.position.y, b.position.x, bid)
|
|
158
|
+
|
|
159
|
+
def visit(bid: str, stack: frozenset[str]):
|
|
160
|
+
if bid in visited:
|
|
161
|
+
return
|
|
162
|
+
if bid in stack:
|
|
163
|
+
raise ValueError(f"cycle detected at block {bid}")
|
|
164
|
+
for pred in sorted(self.predecessors(bid), key=sort_key):
|
|
165
|
+
visit(pred, stack | {bid})
|
|
166
|
+
visited.add(bid)
|
|
167
|
+
order.append(bid)
|
|
168
|
+
|
|
169
|
+
for bid in sorted(self.blocks, key=sort_key):
|
|
170
|
+
visit(bid, frozenset())
|
|
171
|
+
return order
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from .base import LLM_PROVIDER_REGISTRY, ColumnInfo, DraftContext, DraftResult, LLMProvider, get_provider, register_provider
|
|
2
|
+
from .claude_cli_provider import ClaudeCliProvider
|
|
3
|
+
from .gemini_provider import GeminiProvider
|
|
4
|
+
from .lmstudio_provider import LMStudioProvider
|
|
5
|
+
from .openai_provider import OpenAIProvider
|
|
6
|
+
from .stub_provider import StubProvider
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"LLM_PROVIDER_REGISTRY",
|
|
10
|
+
"ColumnInfo",
|
|
11
|
+
"DraftContext",
|
|
12
|
+
"DraftResult",
|
|
13
|
+
"LLMProvider",
|
|
14
|
+
"get_provider",
|
|
15
|
+
"register_provider",
|
|
16
|
+
"StubProvider",
|
|
17
|
+
"ClaudeCliProvider",
|
|
18
|
+
"LMStudioProvider",
|
|
19
|
+
"OpenAIProvider",
|
|
20
|
+
"GeminiProvider",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
from .anthropic_provider import AnthropicProvider # noqa: F401
|
|
25
|
+
|
|
26
|
+
__all__.append("AnthropicProvider")
|
|
27
|
+
except ImportError:
|
|
28
|
+
pass # `anthropic` package not installed; the "anthropic" provider name is unavailable until it is.
|